Import from old repository

plaso/output/__init__.py

@@ -0,0 +1,34 @@
+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+# Copyright 2012 The Plaso Project Authors.
+# Please see the AUTHORS file for details on individual authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""This file contains an import statement for each output plugin."""
+
+from plaso.output import dynamic
+try:
+  from plaso.output import elastic
+except ImportError:
+  pass
+from plaso.output import json_out
+from plaso.output import l2t_csv
+from plaso.output import l2t_tln
+try:
+  from plaso.output import mysql_4n6
+except ImportError:
+  pass
+from plaso.output import pstorage
+from plaso.output import rawpy
+from plaso.output import sqlite_4n6
+from plaso.output import tln

plaso/output/dynamic.py

@@ -0,0 +1,300 @@
+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+#
+# Copyright 2013 The Plaso Project Authors.
+# Please see the AUTHORS file for details on individual authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Contains a formatter for a dynamic output module for plaso."""
+
+import logging
+import re
+
+from plaso.formatters import manager as formatters_manager
+from plaso.lib import errors
+from plaso.lib import output
+from plaso.lib import timelib
+from plaso.output import helper
+
+
+class Dynamic(output.FileLogOutputFormatter):
+  """Dynamic selection of fields for a separated value output format."""
+
+  FORMAT_ATTRIBUTE_RE = re.compile('{([^}]+)}')
+
+  # A dict containing mappings between "special" attributes and
+  # how they should be calculated and presented.
+  # They should be documented here:
+  #   http://plaso.kiddaland.net/usage/psort/output
+  SPECIAL_HANDLING = {
+      'date': 'ParseDate',
+      'datetime': 'ParseDateTime',
+      'description': 'ParseMessage',
+      'description_short': 'ParseMessageShort',
+      'host': 'ParseHostname',
+      'hostname': 'ParseHostname',
+      'inode': 'ParseInode',
+      'macb': 'ParseMacb',
+      'message': 'ParseMessage',
+      'message_short': 'ParseMessageShort',
+      'source': 'ParseSourceShort',
+      'sourcetype': 'ParseSource',
+      'source_long': 'ParseSource',
+      'tag': 'ParseTag',
+      'time': 'ParseTime',
+      'timezone': 'ParseZone',
+      'type': 'ParseTimestampDescription',
+      'user': 'ParseUsername',
+      'username': 'ParseUsername',
+      'zone': 'ParseZone',
+  }
+
+  def ParseTimestampDescription(self, event_object):
+    """Return the timestamp description."""
+    return getattr(event_object, 'timestamp_desc', '-')
+
+  def ParseTag(self, event_object):
+    """Return tagging information."""
+    tag = getattr(event_object, 'tag', None)
+
+    if not tag:
+      return u'-'
+
+    return u' '.join(tag.tags)
+
+  def ParseSource(self, event_object):
+    """Return the source string."""
+    # TODO: move this to an output module interface.
+    event_formatter = formatters_manager.EventFormatterManager.GetFormatter(
+        event_object)
+    if not event_formatter:
+      raise errors.NoFormatterFound(
+          u'Unable to find no event formatter for: {0:s}.'.format(
+              event_object.DATA_TYPE))
+
+    _, source = event_formatter.GetSources(event_object)
+    return source
+
+  def ParseSourceShort(self, event_object):
+    """Return the source string."""
+    # TODO: move this to an output module interface.
+    event_formatter = formatters_manager.EventFormatterManager.GetFormatter(
+        event_object)
+    if not event_formatter:
+      raise errors.NoFormatterFound(
+          u'Unable to find no event formatter for: {0:s}.'.format(
+              event_object.DATA_TYPE))
+
+    source, _ = event_formatter.GetSources(event_object)
+    return source
+
+  def ParseZone(self, _):
+    """Return a timezone."""
+    return self.zone
+
+  def ParseDate(self, event_object):
+    """Return a date string from a timestamp value."""
+    try:
+      date_use = timelib.Timestamp.CopyToDatetime(
+          event_object.timestamp, self.zone, raise_error=True)
+    except OverflowError as exception:
+      logging.error((
+          u'Unable to copy {0:d} into a human readable timestamp with error: '
+          u'{1:s}. Event {2:d}:{3:d} triggered the exception.').format(
+              event_object.timestamp, exception,
+              getattr(event_object, 'store_number', u''),
+              getattr(event_object, 'store_index', u'')))
+      return u'0000-00-00'
+    return u'{0:04d}-{1:02d}-{2:02d}'.format(
+        date_use.year, date_use.month, date_use.day)
+
+  def ParseDateTime(self, event_object):
+    """Return a datetime object from a timestamp, in an ISO format."""
+    try:
+      return timelib.Timestamp.CopyToIsoFormat(
+          event_object.timestamp, timezone=self.zone, raise_error=True)
+
+    except OverflowError as exception:
+      logging.error((
+          u'Unable to copy {0:d} into a human readable timestamp with error: '
+          u'{1:s}. Event {2:d}:{3:d} triggered the exception.').format(
+              event_object.timestamp, exception,
+              getattr(event_object, 'store_number', u''),
+              getattr(event_object, 'store_index', u'')))
+      return u'0000-00-00T00:00:00'
+
+  def ParseTime(self, event_object):
+    """Return a timestamp string from an integer timestamp value."""
+    try:
+      date_use = timelib.Timestamp.CopyToDatetime(
+          event_object.timestamp, self.zone, raise_error=True)
+    except OverflowError as exception:
+      logging.error((
+          u'Unable to copy {0:d} into a human readable timestamp with error: '
+          u'{1:s}. Event {2:d}:{3:d} triggered the exception.').format(
+              event_object.timestamp, exception,
+              getattr(event_object, 'store_number', u''),
+              getattr(event_object, 'store_index', u'')))
+      return u'00:00:00'
+    return u'{0:02d}:{1:02d}:{2:02d}'.format(
+        date_use.hour, date_use.minute, date_use.second)
+
+  def ParseHostname(self, event_object):
+    """Return a hostname."""
+    hostname = getattr(event_object, 'hostname', '')
+    if self.store:
+      if not hostname:
+        hostname = self._hostnames.get(event_object.store_number, '-')
+
+    return hostname
+
+  # TODO: move this into a base output class.
+  def ParseUsername(self, event_object):
+    """Determines an username based on an event and extracted information.
+
+    Uses the extracted information from the pre processing information and the
+    event object itself to determine an username.
+
+    Args:
+      event_object: The event object (instance of EventObject).
+
+    Returns:
+      An Unicode string containing the username, or - if none found.
+    """
+    username = getattr(event_object, u'username', u'-')
+    if self.store:
+      pre_obj = self._preprocesses.get(event_object.store_number)
+      if pre_obj:
+        check_user = pre_obj.GetUsernameById(username)
+
+        if check_user != u'-':
+          username = check_user
+
+    if username == '-' and hasattr(event_object, u'user_sid'):
+      if not pre_obj:
+        return getattr(event_object, u'user_sid', u'-')
+
+      return pre_obj.GetUsernameById(
+          getattr(event_object, u'user_sid', u'-'))
+
+    return username
+
+  def ParseMessage(self, event_object):
+    """Return the message string from the EventObject.
+
+    Args:
+      event_object: The event object (EventObject).
+
+    Raises:
+      errors.NoFormatterFound: If no formatter for that event is found.
+    """
+    # TODO: move this to an output module interface.
+    event_formatter = formatters_manager.EventFormatterManager.GetFormatter(
+        event_object)
+    if not event_formatter:
+      raise errors.NoFormatterFound(
+          u'Unable to find no event formatter for: {0:s}.'.format(
+              event_object.DATA_TYPE))
+
+    msg, _ = event_formatter.GetMessages(event_object)
+    return msg
+
+  def ParseMessageShort(self, event_object):
+    """Return the message string from the EventObject.
+
+    Args:
+      event_object: The event object (EventObject).
+
+    Raises:
+      errors.NoFormatterFound: If no formatter for that event is found.
+    """
+    # TODO: move this to an output module interface.
+    event_formatter = formatters_manager.EventFormatterManager.GetFormatter(
+        event_object)
+    if not event_formatter:
+      raise errors.NoFormatterFound(
+          u'Unable to find no event formatter for: {0:s}.'.format(
+              event_object.DATA_TYPE))
+
+    _, msg_short = event_formatter.GetMessages(event_object)
+    return msg_short
+
+  def ParseInode(self, event_object):
+    """Return an inode number."""
+    inode = getattr(event_object, 'inode', '-')
+    if inode == '-':
+      if hasattr(event_object, 'pathspec') and hasattr(
+          event_object.pathspec, 'image_inode'):
+        inode = event_object.pathspec.image_inode
+
+    return inode
+
+  def ParseMacb(self, event_object):
+    """Return a legacy MACB representation."""
+    return helper.GetLegacy(event_object)
+
+  def Start(self):
+    """Returns a header for the output."""
+    # Start by finding out which fields are to be used.
+    self.fields = []
+
+    if self._filter:
+      self.fields = self._filter.fields
+      self.separator = self._filter.separator
+    else:
+      self.separator = u','
+
+    if not self.fields:
+      # TODO: Evaluate which fields should be included by default.
+      self.fields = [
+          'datetime', 'timestamp_desc', 'source', 'source_long',
+          'message', 'parser', 'display_name', 'tag', 'store_number',
+          'store_index']
+
+    if self.store:
+      self._hostnames = helper.BuildHostDict(self.store)
+      self._preprocesses = {}
+      for info in self.store.GetStorageInformation():
+        if hasattr(info, 'store_range'):
+          for store_number in range(info.store_range[0], info.store_range[1]):
+            self._preprocesses[store_number] = info
+
+    self.filehandle.WriteLine('{0:s}\n'.format(
+        self.separator.join(self.fields)))
+
+  def WriteEvent(self, event_object):
+    """Write a single event."""
+    try:
+      self.EventBody(event_object)
+    except errors.NoFormatterFound:
+      logging.error(u'Unable to output line, no formatter found.')
+      logging.error(event_object)
+
+  def EventBody(self, event_object):
+    """Formats data as "dynamic" CSV and writes to the filehandle."""
+    row = []
+    for field in self.fields:
+      has_call_back = self.SPECIAL_HANDLING.get(field, None)
+      call_back = None
+      if has_call_back:
+        call_back = getattr(self, has_call_back, None)
+
+      if call_back:
+        row.append(call_back(event_object))
+      else:
+        row.append(getattr(event_object, field, u'-'))
+
+    out_write = u'{0:s}\n'.format(
+        self.separator.join(unicode(x).replace(
+            self.separator, u' ') for x in row))
+    self.filehandle.WriteLine(out_write)

plaso/output/dynamic_test.py

@@ -0,0 +1,131 @@
+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+#
+# Copyright 2013 The Plaso Project Authors.
+# Please see the AUTHORS file for details on individual authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Tests for plaso.output.l2t_csv."""
+
+import StringIO
+import unittest
+
+from plaso.formatters import interface as formatters_interface
+from plaso.lib import event
+from plaso.lib import eventdata
+from plaso.output import dynamic
+
+
+class TestEvent(event.EventObject):
+  DATA_TYPE = 'test:dynamic'
+
+  def __init__(self):
+    super(TestEvent, self).__init__()
+    self.timestamp = 1340821021000000
+    self.timestamp_desc = eventdata.EventTimestamp.CHANGE_TIME
+    self.hostname = 'ubuntu'
+    self.filename = 'log/syslog.1'
+    self.text = (
+        u'Reporter <CRON> PID: 8442 (pam_unix(cron:session): session\n '
+        u'closed for user root)')
+
+
+class TestEventFormatter(formatters_interface.EventFormatter):
+  DATA_TYPE = 'test:dynamic'
+  FORMAT_STRING = u'{text}'
+
+  SOURCE_SHORT = 'LOG'
+  SOURCE_LONG = 'Syslog'
+
+
+class FakeFilter(object):
+  """Provide a fake filter, that defines which fields to use."""
+
+  def __init__(self, fields, separator=u','):
+    self.fields = fields
+    self.separator = separator
+
+
+class DynamicTest(unittest.TestCase):
+  """Test the dynamic output module."""
+
+  def testHeader(self):
+    output = StringIO.StringIO()
+    formatter = dynamic.Dynamic(None, output)
+    correct_line = (
+        'datetime,timestamp_desc,source,source_long,message,parser,'
+        'display_name,tag,store_number,store_index\n')
+
+    formatter.Start()
+    self.assertEquals(output.getvalue(), correct_line)
+
+    output = StringIO.StringIO()
+    formatter = dynamic.Dynamic(None, output, filter_use=FakeFilter(
+        ['date', 'time', 'message', 'hostname', 'filename', 'some_stuff']))
+
+    correct_line = 'date,time,message,hostname,filename,some_stuff\n'
+    formatter.Start()
+    self.assertEquals(output.getvalue(), correct_line)
+
+    output = StringIO.StringIO()
+    formatter = dynamic.Dynamic(None, output, filter_use=FakeFilter(
+        ['date', 'time', 'message', 'hostname', 'filename', 'some_stuff'],
+        '@'))
+
+    correct_line = 'date@time@message@hostname@filename@some_stuff\n'
+    formatter.Start()
+    self.assertEquals(output.getvalue(), correct_line)
+
+  def testEventBody(self):
+    """Test ensures that returned lines returned are fmt CSV as expected."""
+    event_object = TestEvent()
+    output = StringIO.StringIO()
+
+    formatter = dynamic.Dynamic(None, output, filter_use=FakeFilter(
+        ['date', 'time', 'timezone', 'macb', 'source', 'sourcetype', 'type',
+         'user', 'host', 'message_short', 'message', 'filename',
+         'inode', 'notes', 'format', 'extra']))
+
+    formatter.Start()
+    header = (
+        'date,time,timezone,macb,source,sourcetype,type,user,host,'
+        'message_short,message,filename,inode,notes,format,extra\n')
+    self.assertEquals(output.getvalue(), header)
+
+    formatter.EventBody(event_object)
+    correct = (
+        '2012-06-27,18:17:01,UTC,..C.,LOG,Syslog,Metadata Modification Time,-,'
+        'ubuntu,Reporter <CRON> PID: 8442 (pam_unix(cron:session): session '
+        'closed for user root),Reporter <CRON> PID: 8442 '
+        '(pam_unix(cron:session): session closed for user root),log/syslog.1'
+        ',-,-,-,-\n')
+    self.assertEquals(output.getvalue(), header + correct)
+
+    output = StringIO.StringIO()
+    formatter = dynamic.Dynamic(None, output, filter_use=FakeFilter(
+        ['datetime', 'nonsense', 'hostname', 'message']))
+
+    header = 'datetime,nonsense,hostname,message\n'
+    formatter.Start()
+    self.assertEquals(output.getvalue(), header)
+
+    correct = (
+        '2012-06-27T18:17:01+00:00,-,ubuntu,Reporter <CRON> PID: 8442'
+        ' (pam_unix(cron:session): session closed for user root)\n')
+
+    formatter.EventBody(event_object)
+    self.assertEquals(output.getvalue(), header + correct)
+
+
+if __name__ == '__main__':
+  unittest.main()

plaso/output/elastic.py

@@ -0,0 +1,235 @@
+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+#
+# Copyright 2013 The Plaso Project Authors.
+# Please see the AUTHORS file for details on individual authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""An output module that saves data into an ElasticSearch database."""
+
+import logging
+import requests
+import sys
+import uuid
+
+import pyelasticsearch
+
+from plaso.formatters import manager as formatters_manager
+from plaso.lib import output
+from plaso.lib import timelib
+from plaso.output import helper
+
+
+class Elastic(output.LogOutputFormatter):
+  """Saves the events into an ElasticSearch database."""
+
+  # Add configuration data for this output module.
+  ARGUMENTS = [
+      ('--case_name', {
+          'dest': 'case_name',
+          'type': unicode,
+          'help': 'Add a case name. This will be the name of the index in '
+                  'ElasticSearch.',
+          'action': 'store',
+          'default': ''}),
+      ('--document_type', {
+          'dest': 'document_type',
+          'type': unicode,
+          'help': 'Name of the document type. This is the name of the document '
+                  'type that will be used in ElasticSearch.',
+          'action': 'store',
+          'default': ''}),
+      ('--elastic_server_ip', {
+          'dest': 'elastic_server',
+          'type': unicode,
+          'help': (
+              'If the ElasticSearch database resides on a different server '
+              'than localhost this parameter needs to be passed in. This '
+              'should be the IP address or the hostname of the server.'),
+          'action': 'store',
+          'default': '127.0.0.1'}),
+      ('--elastic_port', {
+          'dest': 'elastic_port',
+          'type': int,
+          'help': (
+              'By default ElasticSearch uses the port number 9200, if the '
+              'database is listening on a different port this parameter '
+              'can be defined.'),
+          'action': 'store',
+          'default': 9200})]
+
+  def __init__(
+      self, store, filehandle=sys.stdout, config=None, filter_use=None):
+    """Initializes the Elastic output module."""
+    super(Elastic, self).__init__(store, filehandle, config, filter_use)
+    self._counter = 0
+    self._data = []
+    # TODO: move this to an output module interface.
+    self._formatters_manager = formatters_manager.EventFormatterManager
+
+    elastic_host = getattr(config, 'elastic_server', '127.0.0.1')
+    elastic_port = getattr(config, 'elastic_port', 9200)
+    self._elastic_db = pyelasticsearch.ElasticSearch(
+        u'http://{0:s}:{1:d}'.format(elastic_host, elastic_port))
+
+    case_name = getattr(config, 'case_name', u'')
+    document_type = getattr(config, 'document_type', u'')
+
+    # case_name becomes the index name in Elastic.
+    if case_name:
+      self._index_name = case_name.lower()
+    else:
+      self._index_name = uuid.uuid4().hex
+
+    # Name of the doc_type that holds the plaso events.
+    if document_type:
+      self._doc_type = document_type.lower()
+    else:
+      self._doc_type = u'event'
+
+    # Build up a list of available hostnames in this storage file.
+    self._hostnames = {}
+    self._preprocesses = {}
+
+  def _EventToDict(self, event_object):
+    """Returns a dict built from an EventObject."""
+    ret_dict = event_object.GetValues()
+
+    # Get rid of few attributes that cause issues (and need correcting).
+    if 'pathspec' in ret_dict:
+      del ret_dict['pathspec']
+
+    #if 'tag' in ret_dict:
+    #  del ret_dict['tag']
+    #  tag = getattr(event_object, 'tag', None)
+    #  if tag:
+    #    tags = tag.tags
+    #    ret_dict['tag'] = tags
+    #    if getattr(tag, 'comment', ''):
+    #      ret_dict['comment'] = tag.comment
+    ret_dict['tag'] = []
+
+    # To not overload the index, remove the regvalue index.
+    if 'regvalue' in ret_dict:
+      del ret_dict['regvalue']
+
+    # Adding attributes in that are calculated/derived.
+    # We want to remove millisecond precision (causes some issues in
+    # conversion).
+    ret_dict['datetime'] = timelib.Timestamp.CopyToIsoFormat(
+        timelib.Timestamp.RoundToSeconds(event_object.timestamp),
+        timezone=self.zone)
+    msg, _ = self._formatters_manager.GetMessageStrings(event_object)
+    ret_dict['message'] = msg
+
+    source_type, source = self._formatters_manager.GetSourceStrings(
+        event_object)
+
+    ret_dict['source_short'] = source_type
+    ret_dict['source_long'] = source
+
+    hostname = getattr(event_object, 'hostname', '')
+    if self.store and not not hostname:
+      hostname = self._hostnames.get(event_object.store_number, '-')
+
+    ret_dict['hostname'] = hostname
+
+    # TODO: move this into a base output class.
+    username = getattr(event_object, 'username', '-')
+    if self.store:
+      pre_obj = self._preprocesses.get(event_object.store_number)
+      if pre_obj:
+        check_user = pre_obj.GetUsernameById(username)
+
+        if check_user != '-':
+          username = check_user
+
+    if username == '-' and hasattr(event_object, 'user_sid'):
+      username = getattr(event_object, 'user_sid', '-')
+
+    ret_dict['username'] = username
+
+    return ret_dict
+
+  def EventBody(self, event_object):
+    """Prints out to a filehandle string representation of an EventObject.
+
+    Each EventObject contains both attributes that are considered "reserved"
+    and others that aren't. The 'raw' representation of the object makes a
+    distinction between these two types as well as extracting the format
+    strings from the object.
+
+    Args:
+      event_object: The EventObject.
+    """
+    self._data.append(self._EventToDict(event_object))
+    self._counter += 1
+
+    # Check if we need to flush.
+    if self._counter % 5000 == 0:
+      self._elastic_db.bulk_index(self._index_name, self._doc_type, self._data)
+      self._data = []
+      sys.stdout.write('.')
+      sys.stdout.flush()
+
+  def Start(self):
+    """Create the necessary mapping."""
+    if self.store:
+      self._hostnames = helper.BuildHostDict(self.store)
+      for info in self.store.GetStorageInformation():
+        if hasattr(info, 'store_range'):
+          for store_number in range(info.store_range[0], info.store_range[1]):
+            self._preprocesses[store_number] = info
+
+    mapping = {
+        self._doc_type: {
+            u'_timestamp': {
+                u'enabled': True,
+                u'path': 'datetime',
+                u'format': 'date_time_no_millis'},
+        }
+    }
+    # Check if the mappings exist (only create if not there).
+    try:
+      old_mapping_index = self._elastic_db.get_mapping(self._index_name)
+      old_mapping = old_mapping_index.get(self._index_name, {})
+      if self._doc_type not in old_mapping:
+        self._elastic_db.put_mapping(
+            self._index_name, self._doc_type, mapping=mapping)
+    except (pyelasticsearch.ElasticHttpNotFoundError,
+            pyelasticsearch.exceptions.ElasticHttpError):
+      try:
+        self._elastic_db.create_index(self._index_name, settings={
+            'mappings': mapping})
+      except pyelasticsearch.IndexAlreadyExistsError:
+        raise RuntimeError(u'Unable to created the index')
+    except requests.exceptions.ConnectionError as exception:
+      logging.error(
+          u'Unable to proceed, cannot connect to ElasticSearch backend '
+          u'with error: {0:s}.\nPlease verify connection.'.format(exception))
+      raise RuntimeError(u'Unable to connect to ElasticSearch backend.')
+
+    # pylint: disable=unexpected-keyword-arg
+    self._elastic_db.health(wait_for_status='yellow')
+
+    sys.stdout.write('Inserting data')
+    sys.stdout.flush()
+
+  def End(self):
+    """Flush on last time."""
+    self._elastic_db.bulk_index(self._index_name, self._doc_type, self._data)
+    self._data = []
+    sys.stdout.write('. [DONE]\n')
+    sys.stdout.write('ElasticSearch index name: {0:s}\n'.format(
+        self._index_name))
+    sys.stdout.flush()

plaso/output/helper.py

@@ -0,0 +1,109 @@
+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+# Copyright 2013 The Plaso Project Authors.
+# Please see the AUTHORS file for details on individual authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Contains helper functions for output modules."""
+
+from plaso.lib import eventdata
+
+
+def GetLegacy(evt):
+  """Return a legacy MACB representation of the event."""
+  # TODO: Fix this function when the MFT parser has been implemented.
+  # The filestat parser is somewhat limited.
+  # Also fix this when duplicate entries have been implemented so that
+  # the function actually returns more than a single entry (as in combined).
+  if evt.data_type.startswith('fs:'):
+    letter = evt.timestamp_desc[0]
+
+    if letter == 'm':
+      return 'M...'
+    elif letter == 'a':
+      return '.A..'
+    elif letter == 'c':
+      if evt.timestamp_desc[1] == 'r':
+        return '...B'
+
+      return '..C.'
+    else:
+      return '....'
+
+  # Access time.
+  if evt.timestamp_desc in [
+      eventdata.EventTimestamp.ACCESS_TIME,
+      eventdata.EventTimestamp.ACCOUNT_CREATED,
+      eventdata.EventTimestamp.PAGE_VISITED,
+      eventdata.EventTimestamp.LAST_VISITED_TIME,
+      eventdata.EventTimestamp.START_TIME,
+      eventdata.EventTimestamp.LAST_SHUTDOWN,
+      eventdata.EventTimestamp.LAST_LOGIN_TIME,
+      eventdata.EventTimestamp.LAST_PASSWORD_RESET,
+      eventdata.EventTimestamp.LAST_CONNECTED,
+      eventdata.EventTimestamp.LAST_RUNTIME,
+      eventdata.EventTimestamp.LAST_PRINTED]:
+    return '.A..'
+
+  # Content modification.
+  if evt.timestamp_desc in [
+      eventdata.EventTimestamp.MODIFICATION_TIME,
+      eventdata.EventTimestamp.WRITTEN_TIME,
+      eventdata.EventTimestamp.DELETED_TIME]:
+    return 'M...'
+
+  # Content creation time.
+  if evt.timestamp_desc in [
+      eventdata.EventTimestamp.CREATION_TIME,
+      eventdata.EventTimestamp.ADDED_TIME,
+      eventdata.EventTimestamp.FILE_DOWNLOADED,
+      eventdata.EventTimestamp.FIRST_CONNECTED]:
+    return '...B'
+
+  # Metadata modification.
+  if evt.timestamp_desc in [
+      eventdata.EventTimestamp.CHANGE_TIME,
+      eventdata.EventTimestamp.ENTRY_MODIFICATION_TIME]:
+    return '..C.'
+
+  return '....'
+
+
+def BuildHostDict(storage_object):
+  """Return a dict object from a StorageFile object.
+
+  Build a dict object based on the preprocess objects stored inside
+  a storage file.
+
+  Args:
+    storage_object: The StorageFile object that stores all the EventObjects.
+
+  Returns:
+    A dict object that has the store number as a key and the hostname
+    as the value to that key.
+  """
+  host_dict = {}
+  if not storage_object:
+    return host_dict
+
+  if not hasattr(storage_object, 'GetStorageInformation'):
+    return host_dict
+
+  for info in storage_object.GetStorageInformation():
+    if hasattr(info, 'store_range') and hasattr(info, 'hostname'):
+      for store_number in range(info.store_range[0], info.store_range[1]):
+        # TODO: A bit wasteful, if the range is large we are wasting keys.
+        # Rewrite this logic into a more optimal one.
+        host_dict[store_number] = info.hostname
+
+  return host_dict

plaso/output/json_out.py

@@ -0,0 +1,40 @@
+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+#
+# Copyright 2014 The Plaso Project Authors.
+# Please see the AUTHORS file for details on individual authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""An output module that saves data into a simple JSON format."""
+
+from plaso.lib import output
+from plaso.serializer import json_serializer
+
+
+class Json(output.FileLogOutputFormatter):
+  """Saves the events into a JSON format."""
+
+  def EventBody(self, event_object):
+    """Prints out to a filehandle string representation of an EventObject.
+
+    Each event object contains both attributes that are considered "reserved"
+    and others that aren't. The 'raw' representation of the object makes a
+    distinction between these two types as well as extracting the format
+    strings from the object.
+
+    Args:
+      event_object: The event object (instance of EventObject).
+    """
+    self.filehandle.WriteLine(
+        json_serializer.JsonEventObjectSerializer.WriteSerialized(event_object))
+    self.filehandle.WriteLine(u'\n')

plaso/output/json_out_test.py

@@ -0,0 +1,90 @@
+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+#
+# Copyright 2014 The Plaso Project Authors.
+# Please see the AUTHORS file for details on individual authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Tests for the JSON output class."""
+
+import StringIO
+import unittest
+
+from dfvfs.lib import definitions
+from dfvfs.path import factory as path_spec_factory
+
+from plaso.lib import event
+from plaso.lib import timelib_test
+from plaso.output import json_out
+
+
+class JsonTestEvent(event.EventObject):
+  """Simplified EventObject for testing."""
+  DATA_TYPE = 'test:l2tjson'
+
+  def __init__(self):
+    """Initialize event with data."""
+    super(JsonTestEvent, self).__init__()
+    self.timestamp = timelib_test.CopyStringToTimestamp(
+        '2012-06-27 18:17:01+00:00')
+    self.hostname = u'ubuntu'
+    self.display_name = u'OS: /var/log/syslog.1'
+    self.inode = 12345678
+    self.text = (
+        u'Reporter <CRON> PID: |8442| (pam_unix(cron:session): session\n '
+        u'closed for user root)')
+    self.username = u'root'
+
+    os_path_spec = path_spec_factory.Factory.NewPathSpec(
+        definitions.TYPE_INDICATOR_OS, location=u'/cases/image.dd')
+    self.pathspec = path_spec_factory.Factory.NewPathSpec(
+        definitions.TYPE_INDICATOR_TSK, inode=15, location=u'/var/log/syslog.1',
+        parent=os_path_spec)
+
+
+class JsonOutputTest(unittest.TestCase):
+  """Tests for the JSON outputter."""
+
+  def setUp(self):
+    """Sets up the objects needed for this test."""
+    self.output = StringIO.StringIO()
+    self.formatter = json_out.Json(None, self.output)
+    self.event_object = JsonTestEvent()
+
+  def testStartAndEnd(self):
+    """Test to ensure start and end functions do not add text."""
+    self.formatter.Start()
+    self.assertEquals(self.output.getvalue(), u'')
+    self.formatter.End()
+    self.assertEquals(self.output.getvalue(), u'')
+
+  def testEventBody(self):
+    """Test ensures that returned lines returned are formatted as JSON."""
+
+    expected_string = (
+        '{{"username": "root", "display_name": "OS: /var/log/syslog.1", '
+        '"uuid": "{0:s}", "data_type": "test:l2tjson", '
+        '"timestamp": 1340821021000000, "hostname": "ubuntu", "text": '
+        '"Reporter <CRON> PID: |8442| (pam_unix(cron:session): session\\n '
+        'closed for user root)", "pathspec": "{{\\"type_indicator\\": '
+        '\\"TSK\\", \\"inode\\": 15, \\"location\\": \\"/var/log/syslog.1\\", '
+        '\\"parent\\": \\"{{\\\\\\"type_indicator\\\\\\": \\\\\\"OS\\\\\\", '
+        '\\\\\\"location\\\\\\": \\\\\\"/cases/image.dd\\\\\\"}}\\"}}", '
+        '"inode": 12345678}}\n').format(self.event_object.uuid)
+
+    self.formatter.EventBody(self.event_object)
+    self.assertEquals(self.output.getvalue(), expected_string)
+
+
+if __name__ == '__main__':
+  unittest.main()

plaso/output/l2t_csv.py

@@ -0,0 +1,144 @@
+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+#
+# Copyright 2012 The Plaso Project Authors.
+# Please see the AUTHORS file for details on individual authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Contains functions for outputting as l2t_csv.
+
+Author description at: http://code.google.com/p/log2timeline/wiki/l2t_csv
+"""
+
+import logging
+import re
+
+from plaso.formatters import manager as formatters_manager
+from plaso.lib import errors
+from plaso.lib import output
+from plaso.lib import timelib
+from plaso.lib import utils
+from plaso.output import helper
+
+
+class L2tcsv(output.FileLogOutputFormatter):
+  """CSV format used by log2timeline, with 17 fixed fields."""
+
+  FORMAT_ATTRIBUTE_RE = re.compile('{([^}]+)}')
+
+  def Start(self):
+    """Returns a header for the output."""
+    # Build a hostname and username dict objects.
+    self._hostnames = {}
+    if self.store:
+      self._hostnames = helper.BuildHostDict(self.store)
+      self._preprocesses = {}
+      for info in self.store.GetStorageInformation():
+        if hasattr(info, 'store_range'):
+          for store_number in range(
+              info.store_range[0], info.store_range[1] + 1):
+            self._preprocesses[store_number] = info
+
+    self.filehandle.WriteLine(
+        u'date,time,timezone,MACB,source,sourcetype,type,user,host,short,desc,'
+        u'version,filename,inode,notes,format,extra\n')
+
+  def WriteEvent(self, event_object):
+    """Write a single event."""
+    try:
+      self.EventBody(event_object)
+    except errors.NoFormatterFound:
+      logging.error(u'Unable to output line, no formatter found.')
+      logging.error(event_object)
+
+  def EventBody(self, event_object):
+    """Formats data as l2t_csv and writes to the filehandle from OutputFormater.
+
+    Args:
+      event_object: The event object (EventObject).
+
+    Raises:
+      errors.NoFormatterFound: If no formatter for that event is found.
+    """
+    if not hasattr(event_object, 'timestamp'):
+      return
+
+    # TODO: move this to an output module interface.
+    event_formatter = formatters_manager.EventFormatterManager.GetFormatter(
+        event_object)
+    if not event_formatter:
+      raise errors.NoFormatterFound(
+          u'Unable to find event formatter for: {0:s}.'.format(
+              event_object.DATA_TYPE))
+
+    msg, msg_short = event_formatter.GetMessages(event_object)
+    source_short, source_long = event_formatter.GetSources(event_object)
+
+    date_use = timelib.Timestamp.CopyToDatetime(
+        event_object.timestamp, self.zone)
+    extras = []
+    format_variables = self.FORMAT_ATTRIBUTE_RE.findall(
+        event_formatter.format_string)
+    for key in event_object.GetAttributes():
+      if key in utils.RESERVED_VARIABLES or key in format_variables:
+        continue
+      # Force a string conversion since some of the extra attributes
+      # can be numbers or bools.
+      value = getattr(event_object, key)
+      extras.append(u'{0:s}: {1!s} '.format(key, value))
+    extra = ' '.join(extras)
+
+    inode = getattr(event_object, 'inode', '-')
+    if inode == '-':
+      if hasattr(event_object, 'pathspec') and hasattr(
+          event_object.pathspec, 'image_inode'):
+        inode = event_object.pathspec.image_inode
+
+    hostname = getattr(event_object, 'hostname', u'')
+
+    # TODO: move this into a base output class.
+    username = getattr(event_object, 'username', u'-')
+    if self.store:
+      if not hostname:
+        hostname = self._hostnames.get(event_object.store_number, u'-')
+
+      pre_obj = self._preprocesses.get(event_object.store_number)
+      if pre_obj:
+        check_user = pre_obj.GetUsernameById(username)
+        if check_user != '-':
+          username = check_user
+
+    row = (
+        '{0:02d}/{1:02d}/{2:04d}'.format(
+            date_use.month, date_use.day, date_use.year),
+        '{0:02d}:{1:02d}:{2:02d}'.format(
+            date_use.hour, date_use.minute, date_use.second),
+        self.zone,
+        helper.GetLegacy(event_object),
+        source_short,
+        source_long,
+        getattr(event_object, 'timestamp_desc', u'-'),
+        username,
+        hostname,
+        msg_short,
+        msg,
+        '2',
+        getattr(event_object, 'display_name', u'-'),
+        inode,
+        getattr(event_object, 'notes', u'-'),  # Notes field placeholder.
+        getattr(event_object, 'parser', u'-'),
+        extra.replace('\n', u'-').replace('\r', u''))
+
+    out_write = u'{0:s}\n'.format(
+        u','.join(unicode(x).replace(',', u' ') for x in row))
+    self.filehandle.WriteLine(out_write)

plaso/output/l2t_csv_test.py

@@ -0,0 +1,95 @@
+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+#
+# Copyright 2012 The Plaso Project Authors.
+# Please see the AUTHORS file for details on individual authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Tests for the L2tCsv output class."""
+
+import StringIO
+import unittest
+
+from plaso.formatters import interface as formatters_interface
+from plaso.lib import event
+from plaso.lib import eventdata
+from plaso.output import l2t_csv
+
+
+class L2tTestEvent(event.EventObject):
+  """Simplified EventObject for testing."""
+  DATA_TYPE = 'test:l2t_csv'
+
+  def __init__(self):
+    """Initialize event with data."""
+    super(L2tTestEvent, self).__init__()
+    self.timestamp = 1340821021000000
+    self.timestamp_desc = eventdata.EventTimestamp.WRITTEN_TIME
+    self.hostname = u'ubuntu'
+    self.filename = u'log/syslog.1'
+    self.display_name = u'log/syslog.1'
+    self.some_additional_foo = True
+    self.my_number = 123
+    self.text = (
+        u'Reporter <CRON> PID: 8442 (pam_unix(cron:session): session\n '
+        u'closed for user root)')
+
+
+class L2tTestEventFormatter(formatters_interface.EventFormatter):
+  """Formatter for the test event."""
+  DATA_TYPE = 'test:l2t_csv'
+  FORMAT_STRING = u'{text}'
+
+  SOURCE_SHORT = 'LOG'
+  SOURCE_LONG = 'Syslog'
+
+
+class L2tCsvTest(unittest.TestCase):
+  """Contains tests to validate the L2tCSV outputter."""
+  def setUp(self):
+    self.output = StringIO.StringIO()
+    self.formatter = l2t_csv.L2tcsv(None, self.output)
+    self.event_object = L2tTestEvent()
+
+  def testStart(self):
+    """Test ensures header line is outputted as expected."""
+
+    correct_line = (
+        u'date,time,timezone,MACB,source,sourcetype,type,user,host,short,desc,'
+        u'version,filename,inode,notes,format,extra\n')
+
+    self.formatter.Start()
+    self.assertEquals(self.output.getvalue(), correct_line)
+
+  def testEventBody(self):
+    """Test ensures that returned lines returned are formatted as L2tCSV."""
+
+    self.formatter.EventBody(self.event_object)
+    correct = (
+        u'06/27/2012,18:17:01,UTC,M...,LOG,Syslog,Content Modification Time,-,'
+        u'ubuntu,Reporter <CRON> PID: 8442 (pam_unix(cron:session): session '
+        u'closed for user root),Reporter <CRON> PID: 8442 '
+        u'(pam_unix(cron:session): '
+        u'session closed for user root),2,log/syslog.1,-,-,-,my_number: 123  '
+        u'some_additional_foo: True \n')
+    self.assertEquals(self.output.getvalue(), correct)
+
+  def testEventBodyNoExtraCommas(self):
+    """Test ensures that the only commas returned are the 16 delimeters."""
+
+    self.formatter.EventBody(self.event_object)
+    self.assertEquals(self.output.getvalue().count(u','), 16)
+
+
+if __name__ == '__main__':
+  unittest.main()

plaso/output/l2t_tln.py

@@ -0,0 +1,122 @@
+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+#
+# Copyright 2014 The Plaso Project Authors.
+# Please see the AUTHORS file for details on individual authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Contains a class for outputting in the l2tTLN format.
+
+l2tTLN is TLN as expanded in L2T 0.65 to 7 fields:
+
+https://code.google.com/p/log2timeline/source/browse/lib/Log2t/output/tln.pm
+
+Fields:
+  Time - 32 bit Unix epoch.
+  Source - The plugin that produced the data.
+  Host - The source host system.
+  User - The user associated with the data.
+  Description - Message string describing the data.
+  TZ - L2T 0.65 field. Timezone of the event.
+  Notes - L2T 0.65 field. Optional notes field or filename and inode.
+"""
+
+import logging
+
+from plaso.formatters import manager as formatters_manager
+from plaso.lib import errors
+from plaso.lib import output
+from plaso.lib import timelib
+from plaso.output import helper
+
+
+class L2ttln(output.FileLogOutputFormatter):
+  """Extended seven field pipe delimited TLN; L2T 0.65 style."""
+
+  DELIMITER = u'|'
+
+  def Start(self):
+    """Returns a header for the output."""
+    # Build a hostname and username dict objects.
+    self._hostnames = {}
+    if self.store:
+      self._hostnames = helper.BuildHostDict(self.store)
+      self._preprocesses = {}
+      for info in self.store.GetStorageInformation():
+        if hasattr(info, 'store_range'):
+          for store_number in range(
+              info.store_range[0], info.store_range[1] + 1):
+            self._preprocesses[store_number] = info
+    self.filehandle.WriteLine(u'Time|Source|Host|User|Description|TZ|Notes\n')
+
+  def WriteEvent(self, event_object):
+    """Write a single event."""
+    try:
+      self.EventBody(event_object)
+    except errors.NoFormatterFound:
+      logging.error(u'Unable to output line, no formatter found.')
+      logging.error(event_object.GetString())
+
+  def EventBody(self, event_object):
+    """Formats data as TLN and writes to the filehandle from OutputFormater.
+
+    Args:
+      event_object: The event object (EventObject).
+
+    Raises:
+      errors.NoFormatterFound: If no formatter for that event is found.
+    """
+    if not hasattr(event_object, 'timestamp'):
+      return
+
+    # TODO: move this to an output module interface.
+    event_formatter = formatters_manager.EventFormatterManager.GetFormatter(
+        event_object)
+    if not event_formatter:
+      raise errors.NoFormatterFound(
+          u'Unable to find event formatter for: {0:s}.'.format(
+              event_object.DATA_TYPE))
+
+    msg, _ = event_formatter.GetMessages(event_object)
+    source_short, _ = event_formatter.GetSources(event_object)
+
+    date_use = timelib.Timestamp.CopyToPosix(event_object.timestamp)
+    hostname = getattr(event_object, 'hostname', u'')
+    username = getattr(event_object, 'username', u'')
+
+    if self.store:
+      if not hostname:
+        hostname = self._hostnames.get(event_object.store_number, '')
+
+      pre_obj = self._preprocesses.get(event_object.store_number)
+      if pre_obj:
+        check_user = pre_obj.GetUsernameById(username)
+        if check_user != '-':
+          username = check_user
+
+    notes = getattr(event_object, 'notes', u'')
+    if not notes:
+      notes = u'File: {0:s} inode: {1!s}'.format(
+          getattr(event_object, 'display_name', u''),
+          getattr(event_object, 'inode', u''))
+
+    out_write = u'{0!s}|{1:s}|{2:s}|{3:s}|{4:s}|{5:s}|{6!s}\n'.format(
+        date_use,
+        source_short.replace(self.DELIMITER, u' '),
+        hostname.replace(self.DELIMITER, u' '),
+        username.replace(self.DELIMITER, u' '),
+        msg.replace(self.DELIMITER, u' '),
+        self.zone,
+        notes.replace(self.DELIMITER, u' '))
+
+    self.filehandle.WriteLine(out_write)

plaso/output/l2t_tln_test.py

@@ -0,0 +1,86 @@
+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+#
+# Copyright 2014 The Plaso Project Authors.
+# Please see the AUTHORS file for details on individual authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Tests for the l2tTLN output class."""
+
+import StringIO
+import unittest
+
+from plaso.formatters import interface as formatters_interface
+from plaso.lib import event
+from plaso.output import l2t_tln
+
+
+class TlnTestEvent(event.EventObject):
+  """Simplified EventObject for testing."""
+  DATA_TYPE = 'test:tln'
+
+  def __init__(self):
+    """Initialize event with data."""
+    super(TlnTestEvent, self).__init__()
+    self.timestamp = 1340821021000000
+    self.hostname = u'ubuntu'
+    self.display_name = u'OS: log/syslog.1'
+    self.inode = 12345678
+    self.text = (
+        u'Reporter <CRON> PID: |8442| (pam_unix(cron:session): session\n '
+        u'closed for user root)')
+    self.username = u'root'
+
+
+class TlnTestEventFormatter(formatters_interface.EventFormatter):
+  """Formatter for the test event."""
+  DATA_TYPE = 'test:tln'
+  FORMAT_STRING = u'{text}'
+  SOURCE_SHORT = 'LOG'
+  SOURCE_LONG = 'Syslog'
+
+
+class L2TTlnTest(unittest.TestCase):
+  """Tests for the TLN outputter."""
+
+  def setUp(self):
+    """Sets up the objects needed for this test."""
+    self.output = StringIO.StringIO()
+    self.formatter = l2t_tln.L2ttln(None, self.output)
+    self.event_object = TlnTestEvent()
+
+  def testStart(self):
+    """Test ensures header line is outputted as expected."""
+    correct_line = u'Time|Source|Host|User|Description|TZ|Notes\n'
+
+    self.formatter.Start()
+    self.assertEquals(self.output.getvalue(), correct_line)
+
+  def testEventBody(self):
+    """Test ensures that returned lines returned are formatted as TLN."""
+
+    self.formatter.EventBody(self.event_object)
+    correct = (u'1340821021|LOG|ubuntu|root|Reporter <CRON> PID:  8442  '
+               u'(pam_unix(cron:session): session closed for user root)|UTC'
+               u'|File: OS: log/syslog.1 inode: 12345678\n')
+    self.assertEquals(self.output.getvalue(), correct)
+
+  def testEventBodyNoStrayPipes(self):
+    """Test ensures that the only pipes are the six field delimiters."""
+
+    self.formatter.EventBody(self.event_object)
+    self.assertEquals(self.output.getvalue().count(u'|'), 6)
+
+
+if __name__ == '__main__':
+  unittest.main()

plaso/output/mysql_4n6.py

@@ -0,0 +1,402 @@
+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+#
+# Copyright 2012 The Plaso Project Authors.
+# Please see the AUTHORS file for details on individual authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import logging
+import re
+import sys
+
+import MySQLdb
+
+from plaso import formatters
+from plaso.formatters import interface as formatters_interface
+from plaso.formatters import manager as formatters_manager
+from plaso.lib import errors
+from plaso.lib import output
+from plaso.lib import timelib
+from plaso.lib import utils
+from plaso.output import helper
+
+
+__author__ = 'David Nides (david.nides@gmail.com)'
+
+
+class Mysql4n6(output.LogOutputFormatter):
+  """Contains functions for outputting as 4n6time MySQL database."""
+
+  FORMAT_ATTRIBUTE_RE = re.compile('{([^}]+)}')
+
+  META_FIELDS = ['sourcetype', 'source', 'user', 'host', 'MACB',
+                 'color', 'type', 'record_number']
+
+  ARGUMENTS = [
+      ('--db_user', {
+          'dest': 'db_user',
+          'type': unicode,
+          'help': 'Defines the database user.',
+          'metavar': 'USERNAME',
+          'action': 'store',
+          'default': 'root'}),
+      ('--db_host', {
+          'dest': 'db_host',
+          'metavar': 'HOSTNAME',
+          'type': unicode,
+          'help': (
+              'Defines the IP address or the hostname of the database '
+              'server.'),
+          'action': 'store',
+          'default': 'localhost'}),
+      ('--db_pass', {
+          'dest': 'db_pass',
+          'metavar': 'PASSWORD',
+          'type': unicode,
+          'help': 'The password for the database user.',
+          'action': 'store',
+          'default': 'forensic'}),
+      ('--db_name', {
+          'dest': 'db_name',
+          'type': unicode,
+          'help': 'The name of the database to connect to.',
+          'action': 'store',
+          'default': 'log2timeline'}),
+      ('--append', {
+          'dest': 'append',
+          'action': 'store_true',
+          'help': (
+              'Defines whether the intention is to append to an already '
+              'existing database or overwrite it. Defaults to overwrite.'),
+          'default': False}),
+      ('--fields', {
+          'dest': 'fields',
+          'action': 'store',
+          'type': unicode,
+          'nargs': '*',
+          'help': 'Defines which fields should be indexed in the database.',
+          'default': [
+              'host', 'user', 'source', 'sourcetype', 'type', 'datetime',
+              'color']}),
+      ('--evidence', {
+          'dest': 'evidence',
+          'action': 'store',
+          'help': (
+              'Set the evidence field to a specific value, defaults to '
+              'empty.'),
+          'type': unicode,
+          'default': '-'})]
+
+  def __init__(self, store, filehandle=sys.stdout, config=None,
+               filter_use=None):
+    """Constructor for the output module.
+
+    Args:
+      store: The storage object.
+      filehandle: A file-like object that can be written to.
+      config: The configuration object for the module.
+      filter_use: The filter object used.
+    """
+    # TODO: Add a unit test for this output module.
+    super(Mysql4n6, self).__init__(store, filehandle, config, filter_use)
+    # TODO: move this to an output module interface.
+    self._formatters_manager = formatters_manager.EventFormatterManager
+
+    self.set_status = getattr(config, 'set_status', None)
+
+    self.host = getattr(config, 'db_host', 'localhost')
+    self.user = getattr(config, 'db_user', 'root')
+    self.password = getattr(config, 'db_pass', 'forensic')
+    self.dbname = getattr(config, 'db_name', 'log2timeline')
+    self.evidence = getattr(config, 'evidence', '-')
+    self.append = getattr(config, 'append', False)
+    self.fields = getattr(config, 'fields', [
+        'host', 'user', 'source', 'sourcetype', 'type', 'datetime', 'color'])
+
+  def Start(self):
+    """Connect to the database and create the table before inserting."""
+    if self.dbname == '':
+      raise IOError(u'Specify a database name.')
+
+    try:
+      if self.append:
+        self.conn = MySQLdb.connect(self.host, self.user,
+                                    self.password, self.dbname)
+        self.curs = self.conn.cursor()
+      else:
+        self.conn = MySQLdb.connect(self.host, self.user, self.password)
+        self.curs = self.conn.cursor()
+
+      self.conn.set_character_set(u'utf8')
+      self.curs.execute(u'SET NAMES utf8')
+      self.curs.execute(u'SET CHARACTER SET utf8')
+      self.curs.execute(u'SET character_set_connection=utf8')
+      self.curs.execute(u'SET GLOBAL innodb_large_prefix=ON')
+      self.curs.execute(u'SET GLOBAL innodb_file_format=barracuda')
+      self.curs.execute(u'SET GLOBAL innodb_file_per_table=ON')
+      self.curs.execute(
+          u'CREATE DATABASE IF NOT EXISTS {0:s}'.format(self.dbname))
+      self.curs.execute(u'USE {0:s}'.format(self.dbname))
+      # Create tables.
+      self.curs.execute(
+          (u'CREATE TABLE IF NOT EXISTS log2timeline ('
+           u'rowid INT NOT NULL AUTO_INCREMENT, timezone VARCHAR(256), '
+           u'MACB VARCHAR(256), source VARCHAR(256), sourcetype VARCHAR(256), '
+           u'type VARCHAR(256), user VARCHAR(256), host VARCHAR(256), '
+           u'description TEXT, filename VARCHAR(256), inode VARCHAR(256), '
+           u'notes VARCHAR(256), format VARCHAR(256), '
+           u'extra TEXT, datetime datetime, reportnotes VARCHAR(256), '
+           u'inreport VARCHAR(256), tag VARCHAR(256), color VARCHAR(256), '
+           u'offset INT, store_number INT, store_index INT, '
+           u'vss_store_number INT, URL TEXT, '
+           u'record_number VARCHAR(256), event_identifier VARCHAR(256), '
+           u'event_type VARCHAR(256), source_name VARCHAR(256), '
+           u'user_sid VARCHAR(256), computer_name VARCHAR(256), '
+           u'evidence VARCHAR(256), '
+           u'PRIMARY KEY (rowid)) ENGINE=InnoDB ROW_FORMAT=COMPRESSED'))
+      if self.set_status:
+        self.set_status(u'Created table: log2timeline')
+
+      for field in self.META_FIELDS:
+        self.curs.execute(
+            u'CREATE TABLE IF NOT EXISTS l2t_{0}s ({0}s TEXT, frequency INT) '
+            u'ENGINE=InnoDB ROW_FORMAT=COMPRESSED'.format(field))
+        if self.set_status:
+          self.set_status(u'Created table: l2t_{0:s}'.format(field))
+
+      self.curs.execute(
+          u'CREATE TABLE IF NOT EXISTS l2t_tags (tag TEXT) '
+          u'ENGINE=InnoDB ROW_FORMAT=COMPRESSED')
+      if self.set_status:
+        self.set_status(u'Created table: l2t_tags')
+
+      self.curs.execute(
+          u'CREATE TABLE IF NOT EXISTS l2t_saved_query ('
+          u'name TEXT, query TEXT) '
+          u'ENGINE=InnoDB ROW_FORMAT=COMPRESSED')
+      if self.set_status:
+        self.set_status(u'Created table: l2t_saved_query')
+
+      self.curs.execute(
+          u'CREATE TABLE IF NOT EXISTS l2t_disk ('
+          u'disk_type INT, mount_path TEXT, '
+          u'dd_path TEXT, dd_offset TEXT, '
+          u'storage_file TEXT, export_path TEXT) '
+          u'ENGINE=InnoDB ROW_FORMAT=COMPRESSED')
+      self.curs.execute(
+          u'INSERT INTO l2t_disk ('
+          u'disk_type, mount_path, dd_path, '
+          u'dd_offset, storage_file, '
+          u'export_path) VALUES '
+          u'(0, "", "", "", "", "")')
+      if self.set_status:
+        self.set_status(u'Created table: l2t_disk')
+    except MySQLdb.Error as exception:
+      raise IOError(u'Unable to insert into database with error: {0:s}'.format(
+          exception))
+
+    self.count = 0
+
+  def End(self):
+    """Create indices and commit the transaction."""
+    # Build up indices for the fields specified in the args.
+    # It will commit the inserts automatically before creating index.
+    if not self.append:
+      for field_name in self.fields:
+        sql = u'CREATE INDEX {0}_idx ON log2timeline ({0:s})'.format(field_name)
+        self.curs.execute(sql)
+        if self.set_status:
+          self.set_status(u'Created index: {0:s}'.format(field_name))
+
+    # Get meta info and save into their tables.
+    if self.set_status:
+      self.set_status(u'Creating metadata...')
+
+    for field in self.META_FIELDS:
+      vals = self._GetDistinctValues(field)
+      self.curs.execute(u'DELETE FROM l2t_{0:s}s'.format(field))
+      for name, freq in vals.items():
+        self.curs.execute((
+            u'INSERT INTO l2t_{0:s}s ({1:s}s, frequency) '
+            u'VALUES("{2:s}", {3:d}) ').format(field, field, name, freq))
+
+    self.curs.execute(u'DELETE FROM l2t_tags')
+    for tag in self._ListTags():
+      self.curs.execute(
+          u'INSERT INTO l2t_tags (tag) VALUES ("{0:s}")'.format(tag))
+
+    if self.set_status:
+      self.set_status(u'Database created.')
+
+    self.conn.commit()
+    self.curs.close()
+    self.conn.close()
+
+  def _GetDistinctValues(self, field_name):
+    """Query database for unique field types."""
+    self.curs.execute(
+        u'SELECT {0}, COUNT({0}) FROM log2timeline GROUP BY {0}'.format(
+            field_name))
+    res = {}
+    for row in self.curs.fetchall():
+      if row[0] != '':
+        res[row[0]] = int(row[1])
+    return res
+
+  def _ListTags(self):
+    """Query database for unique tag types."""
+    all_tags = []
+    self.curs.execute(
+        u'SELECT DISTINCT tag FROM log2timeline')
+
+    # This cleans up the messy SQL return.
+    for tag_row in self.curs.fetchall():
+      tag_string = tag_row[0]
+      if tag_string:
+        tags = tag_string.split(',')
+        for tag in tags:
+          if tag not in all_tags:
+            all_tags.append(tag)
+    return all_tags
+
+  def EventBody(self, event_object):
+    """Formats data as 4n6time database table format and writes to the db.
+
+    Args:
+      event_object: The event object (EventObject).
+
+    Raises:
+      raise errors.NoFormatterFound: If no formatter for this event is found.
+    """
+    if not hasattr(event_object, 'timestamp'):
+      return
+
+    event_formatter = self._formatters_manager.GetFormatter(event_object)
+    if not event_formatter:
+      raise errors.NoFormatterFound(
+          u'Unable to output event, no event formatter found.')
+
+    if (isinstance(
+        event_formatter, formatters.winreg.WinRegistryGenericFormatter) and
+        event_formatter.FORMAT_STRING.find('<|>') == -1):
+      event_formatter.FORMAT_STRING = u'[{keyname}]<|>{text}<|>'
+
+    elif isinstance(
+        event_formatter, formatters_interface.ConditionalEventFormatter):
+      event_formatter.FORMAT_STRING_SEPARATOR = u'<|>'
+
+    elif isinstance(event_formatter, formatters_interface.EventFormatter):
+      event_formatter.format_string = event_formatter.format_string.replace(
+          '}', '}<|>')
+
+    msg, _ = event_formatter.GetMessages(event_object)
+    source_short, source_long = event_formatter.GetSources(event_object)
+
+    date_use = timelib.Timestamp.CopyToDatetime(
+        event_object.timestamp, self.zone)
+    if not date_use:
+      logging.error(u'Unable to process date for entry: {0:s}'.format(msg))
+      return
+    extra = []
+    format_variables = self.FORMAT_ATTRIBUTE_RE.findall(
+        event_formatter.format_string)
+    for key in event_object.GetAttributes():
+      if key in utils.RESERVED_VARIABLES or key in format_variables:
+        continue
+      extra.append(u'{0:s}: {1!s} '.format(
+          key, getattr(event_object, key, None)))
+
+    extra = u' '.join(extra)
+
+    inode = getattr(event_object, 'inode', '-')
+    if inode == '-':
+      if (hasattr(event_object, 'pathspec') and
+          hasattr(event_object.pathspec, 'image_inode')):
+        inode = event_object.pathspec.image_inode
+
+    date_use_string = u'{0:d}-{1:d}-{2:d} {3:d}:{4:d}:{5:d}'.format(
+        date_use.year, date_use.month, date_use.day, date_use.hour,
+        date_use.minute, date_use.second)
+
+    tags = []
+    if hasattr(event_object, 'tag') and hasattr(event_object.tag, 'tags'):
+      tags = event_object.tag.tags
+    else:
+      tags = u''
+
+    taglist = u','.join(tags)
+    row = (
+        str(self.zone),
+        helper.GetLegacy(event_object),
+        source_short,
+        source_long,
+        getattr(event_object, 'timestamp_desc', '-'),
+        getattr(event_object, 'username', '-'),
+        getattr(event_object, 'hostname', '-'),
+        msg,
+        getattr(event_object, 'filename', '-'),
+        inode,
+        getattr(event_object, 'notes', '-'),
+        getattr(event_object, 'parser', '-'),
+        extra,
+        date_use_string,
+        '',
+        '',
+        taglist,
+        '',
+        getattr(event_object, 'offset', 0),
+        event_object.store_number,
+        event_object.store_index,
+        self.GetVSSNumber(event_object),
+        getattr(event_object, 'url', '-'),
+        getattr(event_object, 'record_number', 0),
+        getattr(event_object, 'event_identifier', '-'),
+        getattr(event_object, 'event_type', '-'),
+        getattr(event_object, 'source_name', '-'),
+        getattr(event_object, 'user_sid', '-'),
+        getattr(event_object, 'computer_name', '-'),
+        self.evidence)
+
+    try:
+      self.curs.execute(
+          'INSERT INTO log2timeline(timezone, MACB, source, '
+          'sourcetype, type, user, host, description, filename, '
+          'inode, notes, format, extra, datetime, reportnotes, '
+          'inreport, tag, color, offset, store_number, '
+          'store_index, vss_store_number, URL, record_number, '
+          'event_identifier, event_type, source_name, user_sid, '
+          'computer_name, evidence) VALUES ('
+          '%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, '
+          '%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, '
+          '%s, %s, %s, %s)', row)
+    except MySQLdb.Error as exception:
+      logging.warning(
+          u'Unable to insert into database with error: {0:s}.'.format(
+              exception))
+
+    self.count += 1
+
+    # TODO: Experiment if committing the current transaction
+    # every 10000 inserts is the optimal approach.
+    if self.count % 10000 == 0:
+      self.conn.commit()
+      if self.set_status:
+        self.set_status(u'Inserting event: {0:d}'.format(self.count))
+
+  def GetVSSNumber(self, event_object):
+    """Return the vss_store_number of the event."""
+    if not hasattr(event_object, 'pathspec'):
+      return -1
+
+    return getattr(event_object.pathspec, 'vss_store_number', -1)

plaso/output/pstorage.py

@@ -0,0 +1,62 @@
+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+#
+# Copyright 2013 The Plaso Project Authors.
+# Please see the AUTHORS file for details on individual authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Implements a StorageFile output formatter."""
+
+from plaso.lib import event
+from plaso.lib import output
+from plaso.lib import storage
+from plaso.lib import timelib
+
+
+class Pstorage(output.LogOutputFormatter):
+  """Dumps event objects to a plaso storage file."""
+
+  def Start(self):
+    """Sets up the output storage file."""
+    pre_obj = event.PreprocessObject()
+    pre_obj.collection_information = {'time_of_run': timelib.Timestamp.GetNow()}
+    if hasattr(self._config, 'filter') and self._config.filter:
+      pre_obj.collection_information['filter'] = self._config.filter
+    if hasattr(self._config, 'storagefile') and self._config.storagefile:
+      pre_obj.collection_information[
+          'file_processed'] = self._config.storagefile
+    self._storage = storage.StorageFile(self.filehandle, pre_obj=pre_obj)
+
+  def EventBody(self, event_object):
+    """Add an EventObject protobuf to the storage file.
+
+    Args:
+      proto: The EventObject protobuf.
+    """
+    # Needed due to duplicate removals, if two events
+    # are merged then we'll just pick the first inode value.
+    inode = getattr(event_object, 'inode', None)
+    if type(inode) in (str, unicode):
+      inode_list = inode.split(';')
+      try:
+        new_inode = int(inode_list[0])
+      except (ValueError, IndexError):
+        new_inode = 0
+
+      event_object.inode = new_inode
+
+    self._storage.AddEventObject(event_object)
+
+  def End(self):
+    """Closes the storage file."""
+    self._storage.Close()

plaso/output/pstorage_test.py

@@ -0,0 +1,93 @@
+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+#
+# Copyright 2013 The Plaso Project Authors.
+# Please see the AUTHORS file for details on individual authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Tests for plaso.output.pstorage."""
+
+import os
+import shutil
+import tempfile
+import unittest
+
+from plaso.lib import output
+from plaso.lib import pfilter
+from plaso.lib import storage
+from plaso.output import pstorage   # pylint: disable=unused-import
+
+
+class TempDirectory(object):
+  """A self cleaning temporary directory."""
+
+  def __init__(self):
+    """Initializes the temporary directory."""
+    super(TempDirectory, self).__init__()
+    self.name = u''
+
+  def __enter__(self):
+    """Make this work with the 'with' statement."""
+    self.name = tempfile.mkdtemp()
+    return self.name
+
+  def __exit__(self, unused_type, unused_value, unused_traceback):
+    """Make this work with the 'with' statement."""
+    shutil.rmtree(self.name, True)
+
+
+class PstorageTest(unittest.TestCase):
+  def setUp(self):
+    self.test_filename = os.path.join('test_data', 'psort_test.out')
+
+    # Show full diff results, part of TestCase so does not follow our naming
+    # conventions.
+    self.maxDiff = None
+    pfilter.TimeRangeCache.ResetTimeConstraints()
+
+  def testOutput(self):
+    with TempDirectory() as dirname:
+      dump_file = os.path.join(dirname, 'plaso.db')
+      # Copy events to pstorage dump.
+      with storage.StorageFile(self.test_filename, read_only=True) as store:
+        formatter_cls = output.GetOutputFormatter('Pstorage')
+        formatter = formatter_cls(store, dump_file)
+        with output.EventBuffer(formatter, check_dedups=False) as output_buffer:
+          event_object = formatter.FetchEntry()
+          while event_object:
+            output_buffer.Append(event_object)
+            event_object = formatter.FetchEntry()
+
+      # Make sure original and dump have the same events.
+      original = storage.StorageFile(self.test_filename, read_only=True)
+      dump = storage.StorageFile(dump_file, read_only=True)
+      event_object_original = original.GetSortedEntry()
+      event_object_dump = dump.GetSortedEntry()
+      original_list = []
+      dump_list = []
+
+      while event_object_original:
+        original_list.append(event_object_original.EqualityString())
+        dump_list.append(event_object_dump.EqualityString())
+        event_object_original = original.GetSortedEntry()
+        event_object_dump = dump.GetSortedEntry()
+
+      self.assertFalse(event_object_dump)
+
+      for original_str, dump_str in zip(
+          sorted(original_list), sorted(dump_list)):
+        self.assertEqual(original_str, dump_str)
+
+
+if __name__ == '__main__':
+  unittest.main()

plaso/output/rawpy.py

@@ -0,0 +1,41 @@
+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+# Copyright 2013 The Plaso Project Authors.
+# Please see the AUTHORS file for details on individual authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Represents an EventObject as a string."""
+from plaso.lib import output
+
+
+class Rawpy(output.FileLogOutputFormatter):
+  """Prints out a "raw" interpretation of the EventObject."""
+  # TODO: Revisit the name of this class, perhaps rename it to
+  # something more closely similar to what it is doing now, as in
+  # "native" or something else.
+
+  def EventBody(self, event_object):
+    """Prints out to a filehandle string representation of an EventObject.
+
+    Each EventObject contains both attributes that are considered "reserved"
+    and others that aren't. The 'raw' representation of the object makes a
+    distinction between these two types as well as extracting the format
+    strings from the object.
+
+    Args:
+      event_object: The EventObject.
+    """
+    # TODO: Move the unicode cast into the event object itself, expose
+    # a ToString function or something similar that will send back the
+    # unicode string.
+    self.filehandle.WriteLine(unicode(event_object))

plaso/output/sqlite_4n6.py

@@ -0,0 +1,316 @@
+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+#
+# Copyright 2012 The Plaso Project Authors.
+# Please see the AUTHORS file for details on individual authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import logging
+import os
+import re
+import sys
+
+import sqlite3
+
+from plaso import formatters
+from plaso.formatters import interface as formatters_interface
+from plaso.formatters import manager as formatters_manager
+from plaso.lib import errors
+from plaso.lib import output
+from plaso.lib import timelib
+from plaso.lib import utils
+from plaso.output import helper
+
+
+__author__ = 'David Nides (david.nides@gmail.com)'
+
+
+class Sql4n6(output.LogOutputFormatter):
+  """Saves the data in a SQLite database, used by the tool 4n6Time."""
+
+  FORMAT_ATTRIBUTE_RE = re.compile('{([^}]+)}')
+
+  META_FIELDS = [
+      'sourcetype', 'source', 'user', 'host', 'MACB', 'color', 'type',
+      'record_number']
+
+  def __init__(self, store, filehandle=sys.stdout, config=None,
+               filter_use=None):
+    """Constructor for the output module.
+
+    Args:
+      store: The storage object.
+      filehandle: A file-like object that can be written to.
+      config: The configuration object for the module.
+      filter_use: The filter object used.
+    """
+    # TODO: Add a unit test for this output module.
+    super(Sql4n6, self).__init__(store, filehandle, config, filter_use)
+    # TODO: move this to an output module interface.
+    self._formatters_manager = formatters_manager.EventFormatterManager
+    self.set_status = getattr(config, 'set_status', None)
+
+    # TODO: Revisit handling this outside of plaso.
+    self.dbname = filehandle
+    self.evidence = getattr(config, 'evidence', '-')
+    self.append = getattr(config, 'append', False)
+    self.fields = getattr(config, 'fields', [
+        'host', 'user', 'source', 'sourcetype', 'type', 'datetime', 'color'])
+
+  # Override LogOutputFormatter methods so it won't write to the file
+  # handle any more.
+  def Start(self):
+    """Connect to the database and create the table before inserting."""
+    if self.filehandle == sys.stdout:
+      raise IOError(
+          u'Unable to connect to stdout as database, please specify a file.')
+
+    if (not self.append) and os.path.isfile(self.filehandle):
+      raise IOError((
+          u'Unable to use an already existing file for output '
+          u'[{0:s}]').format(self.filehandle))
+
+    self.conn = sqlite3.connect(self.dbname)
+    self.conn.text_factory = str
+    self.curs = self.conn.cursor()
+
+    # Create table in database.
+    if not self.append:
+      self.curs.execute(
+          ('CREATE TABLE log2timeline (timezone TEXT, '
+           'MACB TEXT, source TEXT, sourcetype TEXT, type TEXT, '
+           'user TEXT, host TEXT, description TEXT, filename TEXT, '
+           'inode TEXT, notes TEXT, format TEXT, extra TEXT, '
+           'datetime datetime, reportnotes TEXT, '
+           'inreport TEXT, tag TEXT, color TEXT, offset INT,'
+           'store_number INT, store_index INT, vss_store_number INT,'
+           'url TEXT, record_number TEXT, event_identifier TEXT, '
+           'event_type TEXT, source_name TEXT, user_sid TEXT, '
+           'computer_name TEXT, evidence TEXT)'))
+      if self.set_status:
+        self.set_status('Created table: log2timeline')
+
+      for field in self.META_FIELDS:
+        self.curs.execute(
+            'CREATE TABLE l2t_{0}s ({0}s TEXT, frequency INT)'.format(field))
+        if self.set_status:
+          self.set_status('Created table: l2t_{0:s}'.format(field))
+
+      self.curs.execute('CREATE TABLE l2t_tags (tag TEXT)')
+      if self.set_status:
+        self.set_status('Created table: l2t_tags')
+
+      self.curs.execute('CREATE TABLE l2t_saved_query (name TEXT, query TEXT)')
+      if self.set_status:
+        self.set_status('Created table: l2t_saved_query')
+
+      self.curs.execute('CREATE TABLE l2t_disk (disk_type INT, mount_path TEXT,'
+                        ' dd_path TEXT, dd_offset TEXT, storage_file TEXT,'
+                        ' export_path TEXT)')
+      self.curs.execute('INSERT INTO l2t_disk (disk_type, mount_path, dd_path,'
+                        'dd_offset, storage_file, export_path) VALUES '
+                        '(0, "", "", "", "", "")')
+      if self.set_status:
+        self.set_status('Created table: l2t_disk')
+
+    self.count = 0
+
+  def End(self):
+    """Create indices and commit the transaction."""
+    # Build up indices for the fields specified in the args.
+    # It will commit the inserts automatically before creating index.
+    if not self.append:
+      for field_name in self.fields:
+        sql = 'CREATE INDEX {0}_idx ON log2timeline ({0})'.format(field_name)
+        self.curs.execute(sql)
+        if self.set_status:
+          self.set_status('Created index: {0:s}'.format(field_name))
+
+    # Get meta info and save into their tables.
+    if self.set_status:
+      self.set_status('Creating metadata...')
+
+    for field in self.META_FIELDS:
+      vals = self._GetDistinctValues(field)
+      self.curs.execute('DELETE FROM l2t_{0:s}s'.format(field))
+      for name, freq in vals.items():
+        self.curs.execute((
+            'INSERT INTO l2t_{0:s}s ({1:s}s, frequency) '
+            'VALUES("{2:s}", {3:d}) ').format(field, field, name, freq))
+    self.curs.execute('DELETE FROM l2t_tags')
+    for tag in self._ListTags():
+      self.curs.execute('INSERT INTO l2t_tags (tag) VALUES (?)', [tag])
+
+    if self.set_status:
+      self.set_status('Database created.')
+
+    self.conn.commit()
+    self.curs.close()
+    self.conn.close()
+
+  def _GetDistinctValues(self, field_name):
+    """Query database for unique field types."""
+    self.curs.execute(
+        u'SELECT {0}, COUNT({0}) FROM log2timeline GROUP BY {0}'.format(
+            field_name))
+    res = {}
+    for row in self.curs.fetchall():
+      if row[0] != '':
+        res[row[0]] = int(row[1])
+    return res
+
+  def _ListTags(self):
+    """Query database for unique tag types."""
+    all_tags = []
+    self.curs.execute(
+        'SELECT DISTINCT tag FROM log2timeline')
+
+    # This cleans up the messy SQL return.
+    for tag_row in self.curs.fetchall():
+      tag_string = tag_row[0]
+      if tag_string:
+        tags = tag_string.split(',')
+        for tag in tags:
+          if tag not in all_tags:
+            all_tags.append(tag)
+    return all_tags
+
+  def StartEvent(self):
+    """Do nothing, just override the parent's StartEvent method."""
+    pass
+
+  def EndEvent(self):
+    """Do nothing, just override the parent's EndEvent method."""
+    pass
+
+  def EventBody(self, event_object):
+    """Formats data as the 4n6time table format and writes it to the database.
+
+    Args:
+      event_object: The event object (EventObject).
+
+    Raises:
+      raise errors.NoFormatterFound: If no event formatter was found.
+    """
+    if 'timestamp' not in event_object.GetAttributes():
+      return
+
+    event_formatter = self._formatters_manager.GetFormatter(event_object)
+    if not event_formatter:
+      raise errors.NoFormatterFound(
+          'Unable to output event, no event formatter found.')
+
+    if (isinstance(
+        event_formatter, formatters.winreg.WinRegistryGenericFormatter) and
+        event_formatter.FORMAT_STRING.find('<|>') == -1):
+      event_formatter.FORMAT_STRING = u'[{keyname}]<|>{text}<|>'
+
+    elif isinstance(
+        event_formatter, formatters_interface.ConditionalEventFormatter):
+      event_formatter.FORMAT_STRING_SEPARATOR = u'<|>'
+
+    elif isinstance(event_formatter, formatters_interface.EventFormatter):
+      event_formatter.format_string = event_formatter.format_string.replace(
+          '}', '}<|>')
+
+    msg, _ = event_formatter.GetMessages(event_object)
+    source_short, source_long = event_formatter.GetSources(event_object)
+
+    date_use = timelib.Timestamp.CopyToDatetime(
+        event_object.timestamp, self.zone)
+    if not date_use:
+      logging.error(u'Unable to process date for entry: {0:s}'.format(msg))
+      return
+    extra = []
+    format_variables = self.FORMAT_ATTRIBUTE_RE.findall(
+        event_formatter.format_string)
+    for key in event_object.GetAttributes():
+      if key in utils.RESERVED_VARIABLES or key in format_variables:
+        continue
+      extra.append(u'{0:s}: {1!s} '.format(
+          key, getattr(event_object, key, None)))
+    extra = u' '.join(extra)
+
+    inode = getattr(event_object, 'inode', '-')
+    if inode == '-':
+      if (hasattr(event_object, 'pathspec') and
+          hasattr(event_object.pathspec, 'image_inode')):
+        inode = event_object.pathspec.image_inode
+
+    date_use_string = u'{0:04d}-{1:02d}-{2:02d} {3:02d}:{4:02d}:{5:02d}'.format(
+        date_use.year, date_use.month, date_use.day, date_use.hour,
+        date_use.minute, date_use.second)
+
+    tags = []
+    if hasattr(event_object, 'tag'):
+      if hasattr(event_object.tag, 'tags'):
+        tags = event_object.tag.tags
+    taglist = ','.join(tags)
+    row = (str(self.zone),
+           helper.GetLegacy(event_object),
+           source_short,
+           source_long,
+           getattr(event_object, 'timestamp_desc', '-'),
+           getattr(event_object, 'username', '-'),
+           getattr(event_object, 'hostname', '-'),
+           msg,
+           getattr(event_object, 'filename', '-'),
+           inode,
+           getattr(event_object, 'notes', '-'),
+           getattr(event_object, 'parser', '-'),
+           extra,
+           date_use_string,
+           '',
+           '',
+           taglist,
+           '',
+           getattr(event_object, 'offset', 0),
+           event_object.store_number,
+           event_object.store_index,
+           GetVSSNumber(event_object),
+           getattr(event_object, 'url', '-'),
+           getattr(event_object, 'record_number', 0),
+           getattr(event_object, 'event_identifier', '-'),
+           getattr(event_object, 'event_type', '-'),
+           getattr(event_object, 'source_name', '-'),
+           getattr(event_object, 'user_sid', '-'),
+           getattr(event_object, 'computer_name', '-'),
+           self.evidence
+          )
+
+    self.curs.execute(
+        ('INSERT INTO log2timeline(timezone, MACB, source, '
+         'sourcetype, type, user, host, description, filename, '
+         'inode, notes, format, extra, datetime, reportnotes, inreport,'
+         'tag, color, offset, store_number, store_index, vss_store_number,'
+         'URL, record_number, event_identifier, event_type,'
+         'source_name, user_sid, computer_name, evidence)'
+         ' VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?,'
+         '?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)'), row)
+
+    self.count += 1
+
+    # Commit the current transaction every 10000 inserts.
+    if self.count % 10000 == 0:
+      self.conn.commit()
+      if self.set_status:
+        self.set_status('Inserting event: {0:d}'.format(self.count))
+
+
+def GetVSSNumber(event_object):
+  """Return the vss_store_number of the event."""
+  if not hasattr(event_object, 'pathspec'):
+    return -1
+
+  return getattr(event_object.pathspec, 'vss_store_number', -1)

plaso/output/tln.py

@@ -0,0 +1,110 @@
+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+#
+# Copyright 2014 The Plaso Project Authors.
+# Please see the AUTHORS file for details on individual authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Contains a class for outputting in a TLN format.
+
+Output module based on TLN as described by:
+http://windowsir.blogspot.com/2010/02/timeline-analysisdo-we-need-standard.html
+
+Fields:
+  Time - 32 bit Unix epoch.
+  Source - The plugin that produced the data.
+  Host - The source host system.
+  User - The user associated with the data.
+  Description - Message string describing the data.
+"""
+
+import logging
+
+from plaso.formatters import manager as formatters_manager
+from plaso.lib import errors
+from plaso.lib import output
+from plaso.lib import timelib
+from plaso.output import helper
+
+
+class Tln(output.FileLogOutputFormatter):
+  """Five field TLN pipe delimited outputter."""
+
+  DELIMITER = u'|'
+
+  def Start(self):
+    """Returns a header for the output."""
+    # Build a hostname and username dict objects.
+    self._hostnames = {}
+    if self.store:
+      self._hostnames = helper.BuildHostDict(self.store)
+      self._preprocesses = {}
+      for info in self.store.GetStorageInformation():
+        if hasattr(info, 'store_range'):
+          for store_number in range(
+              info.store_range[0], info.store_range[1] + 1):
+            self._preprocesses[store_number] = info
+    self.filehandle.WriteLine(u'Time|Source|Host|User|Description\n')
+
+  def WriteEvent(self, event_object):
+    """Write a single event."""
+    try:
+      self.EventBody(event_object)
+    except errors.NoFormatterFound:
+      logging.error(u'Unable to output line, no formatter found.')
+      logging.error(event_object.GetString())
+
+  def EventBody(self, event_object):
+    """Formats data as TLN and writes to the filehandle from OutputFormater.
+
+    Args:
+      event_object: The event object (EventObject).
+
+    Raises:
+      errors.NoFormatterFound: If no formatter for that event is found.
+    """
+    if not hasattr(event_object, 'timestamp'):
+      return
+
+    # TODO: move this to an output module interface.
+    event_formatter = formatters_manager.EventFormatterManager.GetFormatter(
+        event_object)
+    if not event_formatter:
+      raise errors.NoFormatterFound(
+          u'Unable to find event formatter for: {0:s}.'.format(
+              event_object.DATA_TYPE))
+
+    msg, _ = event_formatter.GetMessages(event_object)
+    source_short, _ = event_formatter.GetSources(event_object)
+
+    date_use = timelib.Timestamp.CopyToPosix(event_object.timestamp)
+    hostname = getattr(event_object, 'hostname', u'')
+    username = getattr(event_object, 'username', u'')
+
+    if self.store:
+      if not hostname:
+        hostname = self._hostnames.get(event_object.store_number, u'')
+
+      pre_obj = self._preprocesses.get(event_object.store_number)
+      if pre_obj:
+        check_user = pre_obj.GetUsernameById(username)
+        if check_user != '-':
+          username = check_user
+
+    out_write = u'{0!s}|{1:s}|{2:s}|{3:s}|{4!s}\n'.format(
+        date_use,
+        source_short.replace(self.DELIMITER, u' '),
+        hostname.replace(self.DELIMITER, u' '),
+        username.replace(self.DELIMITER, u' '),
+        msg.replace(self.DELIMITER, u' '))
+    self.filehandle.WriteLine(out_write)

plaso/output/tln_test.py

@@ -0,0 +1,85 @@
+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+#
+# Copyright 2014 The Plaso Project Authors.
+# Please see the AUTHORS file for details on individual authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Tests for the TLN output class."""
+
+import StringIO
+import unittest
+
+from plaso.formatters import interface as formatters_interface
+from plaso.lib import event
+from plaso.output import tln
+
+
+class TlnTestEvent(event.EventObject):
+  """Simplified EventObject for testing."""
+  DATA_TYPE = 'test:l2ttln'
+
+  def __init__(self):
+    """Initialize event with data."""
+    super(TlnTestEvent, self).__init__()
+    self.timestamp = 1340821021000000
+    self.hostname = u'ubuntu'
+    self.display_name = u'OS: log/syslog.1'
+    self.inode = 12345678
+    self.text = (
+        u'Reporter <CRON> PID: |8442| (pam_unix(cron:session): session\n '
+        u'closed for user root)')
+    self.username = u'root'
+
+
+class L2TTlnTestEventFormatter(formatters_interface.EventFormatter):
+  """Formatter for the test event."""
+  DATA_TYPE = 'test:l2ttln'
+  FORMAT_STRING = u'{text}'
+  SOURCE_SHORT = 'LOG'
+  SOURCE_LONG = 'Syslog'
+
+
+class TlnTest(unittest.TestCase):
+  """Tests for the TLN outputter."""
+
+  def setUp(self):
+    """Sets up the objects needed for this test."""
+    self.output = StringIO.StringIO()
+    self.formatter = tln.Tln(None, self.output)
+    self.event_object = TlnTestEvent()
+
+  def testStart(self):
+    """Test ensures header line is outputted as expected."""
+    correct_line = u'Time|Source|Host|User|Description\n'
+
+    self.formatter.Start()
+    self.assertEquals(self.output.getvalue(), correct_line)
+
+  def testEventBody(self):
+    """Test ensures that returned lines returned are formatted as TLN."""
+
+    self.formatter.EventBody(self.event_object)
+    correct = (u'1340821021|LOG|ubuntu|root|Reporter <CRON> PID:  8442  '
+               u'(pam_unix(cron:session): session closed for user root)\n')
+    self.assertEquals(self.output.getvalue(), correct)
+
+  def testEventBodyNoStrayPipes(self):
+    """Test ensures that the only pipes are the four field delimiters."""
+
+    self.formatter.EventBody(self.event_object)
+    self.assertEquals(self.output.getvalue().count(u'|'), 4)
+
+
+if __name__ == '__main__':
+  unittest.main()