plaso-rubanetra/plaso/parsers/rubanetra.py

#!/usr/bin/python
# -*- coding: utf-8 -*-
#
# Copyright 2014 The Plaso Project Authors.
# Please see the AUTHORS file for details on individual authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import os
import logging
from plaso.lib import errors, event, timelib
from plaso.events.time_events import TimestampEvent
from plaso.lib.eventdata import EventTimestamp
from plaso.parsers import interface
from plaso.parsers import manager

try:
  import xml.etree.cElementTree as ElementTree
except ImportError:
  import xml.etree.ElementTree as ElementTree

__author__ = 'Stefan Swerk (stefan_rubanetra@swerk.priv.at)'


class RubanetraXmlParser(interface.BaseParser):
  """ This class represents the python parser-component of the Rubanetra
      project. Currently, it is only capable of parsing files adhering to the
      XML standard and depends on the 'xml.etree' library.
  """
  NAME = 'rubanetra_xml_parser'
  DESCRIPTION = u'Rubanetra XML file parser'
  VERSION = u'0.0.6'

  RUBANETRA_METADATA_FIELDS = frozenset(['implementationVersion', 'implementationTitle', 'implementationVendor'])

  def Parse(self, parser_context, file_entry, parser_chain=None):
    """ Parses a XML file containing Rubanetra produced content.

    :param parser_context: A parser context object (instance of ParserContext).
    :param file_entry: A file entry object (instance of dfvfs.FileEntry).
    :param parser_chain: Optional string containing the parsing chain up to this
                         point. The default is None.
    """
    rubanetra_metadata_fields = set(self.RUBANETRA_METADATA_FIELDS)
    rubanetra_metadata_dict = dict()

    file_handle = None
    try:
      if file_entry is not None and file_entry.IsFile():
        # Open the file read-only.
        file_handle = file_entry.GetFileObject()
      else:
        raise errors.UnableToParseFile(u'Not a valid Rubanetra file.')

      file_size = file_handle.get_size()

      if file_size <= 0:
        raise errors.UnableToParseFile(u'File size: {0:d} bytes is less or equal than 0.'.format(file_size))

      # read from the beginning and check whether this is a XML file
      file_handle.seek(0, os.SEEK_SET)
      # get an iterable xml parser context
      xml_parser_context = ElementTree.iterparse(file_handle, events=('start', 'end'))
      # turn it into an iterator
      xml_parser_context = iter(xml_parser_context)
      # check whether this is a valid XML file
      try:
        xml_parser_context.next()
      except ElementTree.ParseError:
        raise errors.UnableToParseFile(u'Not a valid Rubanetra file (not XML).')

      # Add ourselves to the parser chain, which will be used in all subsequent
      # event creation in this parser.
      parser_chain = self._BuildParserChain(parser_chain)

      # read from the beginning to process metadata
      file_handle.seek(0, os.SEEK_SET)
      # get an iterable xml parser context
      xml_parser_context = ElementTree.iterparse(file_handle, events=('start', 'end'))
      # turn it into an iterator
      xml_parser_context = iter(xml_parser_context)
      # get the root element
      xml_event, xml_root = xml_parser_context.next()

      for xml_event, xml_elem in xml_parser_context:
        if xml_event == 'end':
          # ... process metadata ...
          if xml_elem.tag in rubanetra_metadata_fields:
            rubanetra_metadata_dict[xml_elem.tag] = xml_elem.text
            rubanetra_metadata_fields.discard(xml_elem.tag)
          elif len(rubanetra_metadata_fields) == 0:
            xml_elem.clear()
            xml_root.clear()
            break

          xml_elem.clear()
          xml_root.clear()

      if len(rubanetra_metadata_fields) != 0:
        raise errors.UnableToParseFile(
            u'Unable to verify metadata, required fields {0:s} could not be parsed'.format(rubanetra_metadata_fields))

      self.validate_metadata(rubanetra_metadata_dict)

      # reopen the file handle
      file_handle.seek(0, os.SEEK_SET)

      # get an iterable xml parser context
      xml_parser_context = ElementTree.iterparse(file_handle, events=('start', 'end'))
      # turn it into an iterator
      xml_parser_context = iter(xml_parser_context)
      # get the root element
      xml_event, xml_root = xml_parser_context.next()

      for xml_event, xml_elem in xml_parser_context:
        if xml_event == 'end':
          # ... process activities ...
          if xml_elem.tag == 'activity':
            activity_dict_list = self.element_to_dict(xml_elem).pop('activity')
            event_objects = self.parse_activity(merge_list_of_dicts_to_dict(activity_dict_list))
            parser_context.ProduceEvents(event_objects, parser_chain=parser_chain, file_entry=file_entry)
          xml_root.clear()
    finally:
      if file_handle is not None:
        file_handle.close()


  def parse_activity(self, activity_dict_merged):
    """ Takes a dictionary resembling an arbitrary activity and parses all fields in a recursive manner in order
    to process nested activities as well as ordinary leaf values.
    The following cases are currently handled:
      - Nested and split activities may occur as a list of dictionaries, as long as each activity dict contains
        an 'activityType'
      - Nested activities are allowed to occur under arbitrary key values
      - EventObjects are constructed using reflective access to the event class, i.e. each occurring 'activityType' must
        be mapped via 'activity_type_to_class_dict' and the mapped class must have a constructor that can handle the
        parsing process of a single flat activity using a dict.
      - If there is no mapping of an occurring 'activityType' within 'activity_type_to_class_dict', a BaseActivityEvent
        will be constructed instead.
      - Leaf values that do not represent an entire Activity will not be modified but instead passed as constructor
        argument to the corresponding EventObject implementation.

    :param activity_dict_merged: an arbitrary activity as dictionary, conforming to a certain 'activityType'.
    :return: a list of EventObjects that could be parsed from the given dictionary, corresponding to the
             value of 'activityType', if possible. Otherwise, either an empty list, in case the given dict does not
             contain a valid activity, or None, in case the given argument is not a dict, will be returned.
    """
    if activity_dict_merged is None:
      return None

    event_objects = list()
    if isinstance(activity_dict_merged, list):
      for v in activity_dict_merged:
        if isinstance(v, dict) and 'activityType' in v:
          if len(v) == 1:
            # It is safe to assume that this is a nested and split activity. Therefore break the loop
            # and parse it. This assumption does no longer hold true in case of a scenario where
            # an activity_dict_list contains both, a split activity and another nested activity_dict.
            # Currently the xml parser handles this case by wrapping the nested activity_dict in another
            # list.
            return self.parse_activity(merge_list_of_dicts_to_dict(activity_dict_merged))
          else:
            # this is a nested activity, parse and save objects
            child_evt_objs = self.parse_activity(v)
            if child_evt_objs is not None:
              for e in child_evt_objs:
                if isinstance(e, event.EventObject):
                  event_objects += [e]

        else:
          event_objects += self.parse_activity(v)

      return event_objects

    # is it a leaf value or an actual activity?
    if not isinstance(activity_dict_merged, dict):
      return None


    # it is at least a dict, however, is it an activity or a value?
    activity_type = activity_dict_merged.get('activityType', None)
    if activity_type is not None:  # it is an activity
      event_object_class = activity_type_to_class_dict.get(activity_type, BaseActivityEvent)
      event_objects = [event_object_class(activity_dict_merged)]  # TODO check whether it is an actual class

    # everything that remains may be another activity or an unconsumed leaf value
    for k, v in activity_dict_merged.items():
      child_evt_objs = self.parse_activity(v)
      # currently, the key attribute is not used -> TODO: find a way to link the child-events to its parent?
      if child_evt_objs is not None:
        for e in child_evt_objs:
          if isinstance(e, event.EventObject):
            event_objects += [e]

    return event_objects

  def parse_timestamp_events(self, activity_dict):
    """ Takes a dictionary of dictionaries that must contain at least two keys:
      - 'startInstant', a dictionary that corresponds to a serialized Java Instant object
      - 'endInstant', as above

      This method will produce a dictionary consisting of either one or two JavaInstantEvent objects.

    :param activity_dict: containing the 'startInstant' and 'endInstant' dictionaries
    :return: either one, iff 'startInstant' == 'endInstant', or two JavaInstantEvent-objects inside a dictionary.
    """
    start_instant_dict = merge_list_of_dicts_to_dict(activity_dict.get('startInstant', None))
    end_instant_dict = merge_list_of_dicts_to_dict(activity_dict.get('endInstant', None))

    if start_instant_dict != end_instant_dict:
      instant_dict = dict(startInstant=None, endInstant=None)
      # interval
      start_instant_evt = JavaInstantEvent.from_java_instant_dict(start_instant_dict,
                                                                  EventTimestamp.FIRST_CONNECTED)
      end_instant_evt = JavaInstantEvent.from_java_instant_dict(end_instant_dict,
                                                                EventTimestamp.LAST_CONNECTED)
      instant_dict['startInstant'] = start_instant_evt
      instant_dict['endInstant'] = end_instant_evt
      return instant_dict
    else:
      return dict(startInstant=JavaInstantEvent.from_java_instant_dict(start_instant_dict, u'Pcap time stamp'))

  def element_to_dict(self, elem):
    """ Internal method to transform a XML node to a dictionary.

    :param elem: the XML node
    :return: a dictionary containing the values below 'elem', using 'elem.tag' as respective keys
    """
    return {elem.tag: map(self.element_to_dict, list(elem)) or elem.text}

  def validate_metadata(self, rubanetra_metadata_dict):
    """ Tries to verify that the parsed XML document corresponds to a known version to prevent potential
    issues due to version incompatibility. An exception will be raised if such a case is encountered.

    :param rubanetra_metadata_dict: a dictionary containing the basic Rubanetra metadata values
    """
    if rubanetra_metadata_dict.get('implementationTitle') != u'Rubanetra':
      raise errors.UnableToParseFile(u'Unknown Rubanetra implementation title encountered.')

    version = rubanetra_metadata_dict.get('implementationVersion')
    if version != self.VERSION:
      logging.warning(u'Rubanetra version number mismatch, expected:{0:s}, actual:{1:s}'.format(self.VERSION, version))


  def link_activity(self, event_object_from, event_object_to):
    """ This method is currently unused, however, in case it is necessary to group multiple events,
    a link between those events must be established. Whether a backtracking chain or a forward-chain should be
    established depends entirely on the caller.
    Currently, the UUID of 'event_object_to' will be appended to the list 'related_activity_uuids' of
    'event_object_from'.
    """
    if isinstance(event_object_to, BaseActivityEvent):
      event_object_from.related_activity_uuids.append(event_object_to.uuid)
      # TODO: else error


class BaseActivityEvent(event.EventObject):
  def __init__(self, activity_dict,
               data_type='java:rubanetra:base_activity'):
    """Initializes the base event object.

    Args:
      activity_dict: A dictionary containing all related BaseActivity key/value pairs.
    """
    super(BaseActivityEvent, self).__init__()

    if activity_dict is None:
      raise errors.UnableToParseFile

    self.data_type = data_type
    self.activity_type = activity_dict.pop('activityType', 'BaseActivity')
    self.description = activity_dict.pop('description', None)
    compound_frame_number_dict_list = activity_dict.pop('compoundFrameNumbers', None)
    self.compound_frame_number_list = list()
    if compound_frame_number_dict_list is not None:
      for d in compound_frame_number_dict_list:
        for k, v in d.items():
          self.compound_frame_number_list.append(long(v))  # TODO checks

    self.optional_field_dict = merge_list_of_dicts_to_dict(activity_dict.pop('optionalFields', None))
    self.replaced = string_to_boolean(activity_dict.pop('replaced', None))
    self.source_address = activity_dict.pop('sourceAddressAsString', None)
    self.destination_address = activity_dict.pop('destinationAddressAsString', None)

    start_instant_dict = merge_list_of_dicts_to_dict(activity_dict.get('startInstant', None))
    end_instant_dict = merge_list_of_dicts_to_dict(activity_dict.get('endInstant', None))

    start_instant_evt = None
    if start_instant_dict != end_instant_dict:
      start_instant_evt = JavaInstantEvent.from_java_instant_dict(start_instant_dict,
                                                                  EventTimestamp.FIRST_CONNECTED)
      # interval
      end_instant_evt = JavaInstantEvent.from_java_instant_dict(end_instant_dict,
                                                                EventTimestamp.LAST_CONNECTED)
      self.last_timestamp = end_instant_evt.timestamp if end_instant_evt is not None else None
    else:
      start_instant_evt = JavaInstantEvent.from_java_instant_dict(start_instant_dict, u'Pcap time stamp')
    self.timestamp = self.first_timestamp = start_instant_evt.timestamp if start_instant_evt is not None else None
    self.timestamp_desc = start_instant_evt.timestamp_desc
    self.related_activity_uuids = list()


class PcapActivityEvent(BaseActivityEvent):
  def __init__(self, pcap_activity_dict):
    super(PcapActivityEvent, self).__init__(pcap_activity_dict,
                                            data_type='java:rubanetra:pcap_activity')
    pcap_packet = merge_list_of_dicts_to_dict(pcap_activity_dict.pop('pcapPacket', None))
    if pcap_packet is not None:
      self.pcap_total_size = pcap_packet.pop('totalSize', None)
      self.pcap_frame_number = pcap_packet.pop('frameNumber', None)
      self.pcap_packet_wirelen = pcap_packet.pop('packetWirelen', None)
      self.pcap_header_count = pcap_packet.pop('headerCount', None)


class HttpRequestActivityEvent(BaseActivityEvent):
  def __init__(self, http_request_activity_dict):
    super(HttpRequestActivityEvent, self).__init__(http_request_activity_dict,
                                                   data_type='java:rubanetra:http_request_activity')
    self.http_version = http_request_activity_dict.pop('httpVersion', None)
    self.server_address = http_request_activity_dict.pop('serverAddress', None)
    self.client_address = http_request_activity_dict.pop('clientAddress', None)
    self.http_method = http_request_activity_dict.pop('httpMethod', None)
    self.http_query_string = http_request_activity_dict.pop('httpQueryString', None)
    self.http_query_parameters = http_request_activity_dict.pop('httpQueryParameters', None)
    self.http_request_header_dict = http_request_activity_dict.pop('requestHeaderMap', None)
    self.url = http_request_activity_dict.pop('url', None)

    http_request = merge_list_of_dicts_to_dict(http_request_activity_dict.pop('httpRequest', None))
    if http_request is not None:
      self.orig_http_header = http_request.pop('header', None)
      self.content_type = http_request.pop('contentType', None)
      self.is_response = http_request.pop('response', None)
      self.jnetpcap_http_string = http_request.pop('JNetPcap-HTTP-String', None)

    self.source_address = self.client_address
    self.destination_address = self.server_address


class HttpResponseActivityEvent(BaseActivityEvent):
  def __init__(self, http_response_activity_dict):
    super(HttpResponseActivityEvent, self).__init__(http_response_activity_dict,
                                                    data_type='java:rubanetra:http_response_activity')
    self.http_version = http_response_activity_dict.pop('httpVersion', None)
    self.response_status_code = http_response_activity_dict.pop('responseStatusCode', None)
    self.response_status_line = http_response_activity_dict.pop('responseStatusLine', None)
    self.response_header_dict = http_response_activity_dict.pop('responseHeaderMap', None)

    http_response = merge_list_of_dicts_to_dict(http_response_activity_dict.pop('httpResponse', None))

    if http_response is not None:
      self.orig_http_header = http_response.pop('header', None)
      self.content_type = http_response.pop('contentType', None)
      self.is_response = http_response.pop('response', None)
      self.jnetpcap_http_string = http_response.pop('JNetPcap-HTTP-String', None)


class HttpImageActivityEvent(BaseActivityEvent):
  def __init__(self, http_image_activity_dict):
    super(HttpImageActivityEvent, self).__init__(http_image_activity_dict,
                                                 data_type='java:rubanetra:http_image_activity')

    self.image_type = http_image_activity_dict.pop('imageType', None)
    self.image_path = http_image_activity_dict.pop('imagePath', None)


class DnsActivityEvent(BaseActivityEvent):
  def __init__(self, dns_activity_dict):
    super(DnsActivityEvent, self).__init__(dns_activity_dict,
                                           data_type='java:rubanetra:dns_activity')
    self.question_record_list = dns_activity_dict.pop('questionRecords', None)
    self.answer_record_list = dns_activity_dict.pop('answerRecords', None)
    self.authority_record_list = dns_activity_dict.pop('authorityRecords', None)
    self.additional_record_list = dns_activity_dict.pop('additionalRecords', None)
    self.dns_message_header = merge_list_of_dicts_to_dict(dns_activity_dict.pop('dnsMessageHeader', None))
    self.is_response_bool = string_to_boolean(dns_activity_dict.pop('response', None))


class ArpActivityEvent(BaseActivityEvent):
  def __init__(self, arp_activity_dict):
    super(ArpActivityEvent, self).__init__(arp_activity_dict,
                                           data_type='java:rubanetra:arp_activity')
    self.hardware_type = arp_activity_dict.pop('hardwareType', None)
    self.protocol_type = arp_activity_dict.pop('protocolType', None)
    self.hardware_address_length = arp_activity_dict.pop('hardwareAddressLength', None)
    self.protocol_address_length = arp_activity_dict.pop('protocolAddressLength', None)
    self.sender_mac_address = arp_activity_dict.pop('senderHardwareAddress', None)
    self.target_mac_address = arp_activity_dict.pop('targetHardwareAddress', None)
    self.sender_protocol_address = arp_activity_dict.pop('senderProtocolAddress', None)
    self.target_protocol_address = arp_activity_dict.pop('targetProtocolAddress', None)
    self.jnetpcap_arp = arp_activity_dict.pop('arp', None)


class DhcpActivityEvent(BaseActivityEvent):
  def __init__(self, dhcp_activity_dict):
    super(DhcpActivityEvent, self).__init__(dhcp_activity_dict,
                                            data_type='java:rubanetra:dhcp_activity')
    self.dhcp_message = dhcp_activity_dict.pop('dhcpMessage', None)


class EthernetActivityEvent(BaseActivityEvent):
  def __init__(self, ethernet_activity_dict):
    super(EthernetActivityEvent, self).__init__(ethernet_activity_dict,
                                                data_type='java:rubanetra:ethernet_activity')
    self.source_mac_address = ethernet_activity_dict.pop('sourceMacAddress', None)
    self.destination_mac_address = ethernet_activity_dict.pop('destinationMacAddress', None)
    self.ethernet_type = ethernet_activity_dict.pop('ethernetType', None)
    self.ethernet_type_enum = ethernet_activity_dict.pop('ethernetTypeEnum', None)
    self.jnetpcap_ethernet = ethernet_activity_dict.pop('ethernet', None)


class FtpActivityEvent(BaseActivityEvent):
  def __init__(self, ftp_activity_dict):
    super(FtpActivityEvent, self).__init__(ftp_activity_dict,
                                           data_type='java:rubanetra:ftp_activity')
    self.ftp_type = ftp_activity_dict.pop('ftpActivityType', None)
    self.command = ftp_activity_dict.pop('command', None)
    self.reply = ftp_activity_dict.pop('reply', None)
    self.list = ftp_activity_dict.pop('list', None)


class Icmpv4ActivityEvent(BaseActivityEvent):
  def __init__(self, icmpv4_activity_dict):
    super(Icmpv4ActivityEvent, self).__init__(icmpv4_activity_dict,
                                              data_type='java:rubanetra:icmpv4_activity')
    self.icmp_subtype = icmpv4_activity_dict.pop('icmpSubType', None)
    self.icmp_packet = icmpv4_activity_dict.pop('icmpPacket', None)
    self.icmp_message = icmpv4_activity_dict.pop('icmpMessage', None)
    self.icmp_type = icmpv4_activity_dict.pop('icmpType', None)
    self.icmp_code = icmpv4_activity_dict.pop('icmpCode', None)
    self.source_address = icmpv4_activity_dict.pop('sourceAddress', None)
    self.destination_address = icmpv4_activity_dict.pop('destinationAddress', None)
    self.identifier = icmpv4_activity_dict.pop('identifier', None)
    self.sequence = icmpv4_activity_dict.pop('sequence', None)
    self.jnetpcap_icmp = icmpv4_activity_dict.pop('icmp', None)


class Icmpv6ActivityEvent(BaseActivityEvent):
  def __init__(self, icmpv6_activity_dict):
    super(Icmpv6ActivityEvent, self).__init__(icmpv6_activity_dict,
                                              data_type='java:rubanetra:icmpv6_activity')
    self.icmp_subtype = icmpv6_activity_dict.pop('icmpSubType', None)
    self.icmp_packet = icmpv6_activity_dict.pop('icmpPacket', None)
    self.icmp_message = icmpv6_activity_dict.pop('icmpMessage', None)
    self.icmp_type = icmpv6_activity_dict.pop('icmpType', None)
    self.jnetpcap_icmp = icmpv6_activity_dict.pop('icmp', None)


class IpActivityEvent(BaseActivityEvent):
  def __init__(self, ip_activity_dict):
    super(IpActivityEvent, self).__init__(ip_activity_dict,
                                          data_type='java:rubanetra:ip_activity')
    self.version = ip_activity_dict.pop('version', None)
    self.protocol = ip_activity_dict.pop('protocol', None)
    self.source_address = ip_activity_dict.pop('sourceAddress', None)
    self.destination_address = ip_activity_dict.pop('destinationAddress', None)


class Ipv4ActivityEvent(BaseActivityEvent):
  def __init__(self, ip_activity_dict):
    super(Ipv4ActivityEvent, self).__init__(ip_activity_dict,
                                            data_type='java:rubanetra:ipv4_activity')
    self.internet_header_length = ip_activity_dict.pop('internetHeaderLength', None)
    self.differentiated_services_code_point = ip_activity_dict.pop('differentiatedServicesCodePoint', None)
    self.total_length = ip_activity_dict.pop('totalLength', None)
    self.identification = ip_activity_dict.pop('identification', None)
    self.flags = ip_activity_dict.pop('flags', None)
    self.fragment_offset = ip_activity_dict.pop('fragmentOffset', None)
    self.time_to_live = ip_activity_dict.pop('timeToLive', None)
    self.header_checksum = ip_activity_dict.pop('headerChecksum', None)
    self.options = ip_activity_dict.pop('options', None)
    self.jnetpcap_ip4 = ip_activity_dict.pop('ipv4', None)


class Ipv6ActivityEvent(BaseActivityEvent):
  def __init__(self, ip_activity_dict):
    super(Ipv6ActivityEvent, self).__init__(ip_activity_dict,
                                            data_type='java:rubanetra:ipv6_activity')
    self.traffic_class = ip_activity_dict.pop('trafficClass', None)
    self.flow_label = ip_activity_dict.pop('flowLabel', None)
    self.payload_length = ip_activity_dict.pop('payloadLength', None)
    self.next_header = ip_activity_dict.pop('nextHeader', None)
    self.hop_limit = ip_activity_dict.pop('hopLimit', None)
    self.jnetpcap_ip6 = ip_activity_dict.pop('ipv6', None)
    self.kraken_ip6 = ip_activity_dict.pop('ipv6Packet', None)


class MsnActivityEvent(BaseActivityEvent):
  def __init__(self, activity_dict):
    super(MsnActivityEvent, self).__init__(activity_dict,
                                           data_type='java:rubanetra:msn_activity')
    self.account = activity_dict.pop('account', None)
    self.chat = activity_dict.pop('chat', None)


class NetbiosActivityEvent(BaseActivityEvent):
  def __init__(self, activity_dict):
    super(NetbiosActivityEvent, self).__init__(activity_dict,
                                               data_type='java:rubanetra:netbios_activity')
    self.datagram_packet = activity_dict.pop('datagramPacket', None)
    self.name_packet = activity_dict.pop('namePacket', None)


class Pop3ActivityEvent(BaseActivityEvent):
  def __init__(self, activity_dict):
    super(Pop3ActivityEvent, self).__init__(activity_dict,
                                            data_type='java:rubanetra:pop3_activity')
    self.sub_type = activity_dict.pop('subType', None)
    self.header = activity_dict.pop('header', None)
    self.data = activity_dict.pop('data', None)
    self.command = activity_dict.pop('command', None)
    self.response = activity_dict.pop('response', None)


class SmtpCommandActivityEvent(BaseActivityEvent):
  def __init__(self, activity_dict):
    super(SmtpCommandActivityEvent, self).__init__(activity_dict,
                                                   data_type='java:rubanetra:smtp_command_activity')
    self.command = activity_dict.pop('command', None)
    self.parameter = activity_dict.pop('parameter', None)


class SmtpReplyActivityEvent(BaseActivityEvent):
  def __init__(self, activity_dict):
    super(SmtpReplyActivityEvent, self).__init__(activity_dict,
                                                 data_type='java:rubanetra:smtp_reply_activity')
    self.code = activity_dict.pop('code', None)
    self.message = activity_dict.pop('message', None)


class SmtpSendActivityEvent(BaseActivityEvent):
  def __init__(self, activity_dict):
    super(SmtpSendActivityEvent, self).__init__(activity_dict,
                                                data_type='java:rubanetra:smtp_send_activity')
    self.header = activity_dict.pop('header', None)
    self.data = activity_dict.pop('data', None)


class Snmpv1ActivityEvent(BaseActivityEvent):
  def __init__(self, activity_dict):
    super(Snmpv1ActivityEvent, self).__init__(activity_dict,
                                              data_type='java:rubanetra:snmpv1_activity')
    self.pdu = activity_dict.pop('pdu', None)
    self.source_socket_address = activity_dict.pop('sourceSocketAddress', None)
    self.destination_socket_address = activity_dict.pop('destinationSocketAddress', None)


class Snmpv2ActivityEvent(BaseActivityEvent):
  def __init__(self, activity_dict):
    super(Snmpv2ActivityEvent, self).__init__(activity_dict,
                                              data_type='java:rubanetra:snmpv2_activity')
    self.pdu = activity_dict.pop('pdu', None)
    self.source_socket_address = activity_dict.pop('sourceSocketAddress', None)
    self.destination_socket_address = activity_dict.pop('destinationSocketAddress', None)


class TcpActivityEvent(BaseActivityEvent):
  def __init__(self, activity_dict):
    super(TcpActivityEvent, self).__init__(activity_dict,
                                           data_type='java:rubanetra:tcp_activity')
    self.source_port = activity_dict.pop('sourcePort', None)
    self.destination_port = activity_dict.pop('destinationPort', None)
    self.sequence_number = activity_dict.pop('sequenceNumber', None)
    self.acknowledge_number = activity_dict.pop('acknowledgeNumber', None)
    self.relative_sequence_number = activity_dict.pop('relativeSequenceNumber', None)
    self.relative_acknowledge_number = activity_dict.pop('relativeAcknowledgeNumber', None)
    self.data_offset = activity_dict.pop('dataOffset', None)
    self.control_bits = activity_dict.pop('controlBits', None)
    self.window_size = activity_dict.pop('windowSize', None)
    self.checksum = activity_dict.pop('checksum', None)
    self.urgent_pointer = activity_dict.pop('urgentPointer', None)
    self.tcp_length = activity_dict.pop('tcpLength', None)
    self.options = activity_dict.pop('options', None)
    self.padding = activity_dict.pop('padding', None)
    self.syn = activity_dict.pop('syn', None)
    self.ack = activity_dict.pop('ack', None)
    self.psh = activity_dict.pop('psh', None)
    self.fin = activity_dict.pop('fin', None)
    self.rst = activity_dict.pop('rst', None)
    self.urg = activity_dict.pop('urg', None)
    self.direction = activity_dict.pop('direction', None)
    self.client_state = activity_dict.pop('clientState', None)
    self.server_state = activity_dict.pop('serverState', None)
    self.jnetpcap_tcp = activity_dict.pop('tcp', None)
    self.source_address = activity_dict.pop('sourceAddress', None)
    self.destination_address = activity_dict.pop('destinationAddress', None)
    self.source_socket_address = activity_dict.pop('sourceSocketAddress', None)
    self.destination_socket_address = activity_dict.pop('destinationSocketAddress', None)


class TelnetActivityEvent(BaseActivityEvent):
  def __init__(self, activity_dict):
    super(TelnetActivityEvent, self).__init__(activity_dict,
                                              data_type='java:rubanetra:telnet_activity')
    self.sub_type = activity_dict.pop('subType', None)
    self.command = activity_dict.pop('command', None)
    self.option = activity_dict.pop('option', None)
    self.ansi_mode = activity_dict.pop('ansiMode', None)
    self.arguments = activity_dict.pop('arguments', None)
    self.text = activity_dict.pop('text', None)
    self.title = activity_dict.pop('title', None)


class TlsActivityEvent(BaseActivityEvent):
  def __init__(self, activity_dict):
    super(TlsActivityEvent, self).__init__(activity_dict,
                                           data_type='java:rubanetra:tls_activity')
    self.client_to_server_traffic = activity_dict.pop('clientToServerTraffic', None)
    self.server_to_client_traffic = activity_dict.pop('serverToClientTraffic', None)


class UdpActivityEvent(BaseActivityEvent):
  def __init__(self, activity_dict):
    super(UdpActivityEvent, self).__init__(activity_dict,
                                           data_type='java:rubanetra:udp_activity')
    self.source_port = activity_dict.pop('sourcePort', None)
    self.destination_port = activity_dict.pop('destinationPort', None)
    self.length = activity_dict.pop('length', None)
    self.checksum = activity_dict.pop('checksum', None)
    self.jnetpcap_udp = activity_dict.pop('udp', None)
    self.source_socket_address = activity_dict.pop('sourceSocketAddress', None)
    self.destination_socket_address = activity_dict.pop('destinationSocketAddress', None)


class OpenSSHActivityEvent(BaseActivityEvent):
  def __init__(self, activity_dict):
    super(OpenSSHActivityEvent, self).__init__(activity_dict,
                                               data_type='java:rubanetra:open_ssh_activity')
    self.client_to_server_traffic = activity_dict.pop('clientToServerTraffic', None)
    self.server_to_client_traffic = activity_dict.pop('serverToClientTraffic', None)


class DropboxTlsActivityEvent(BaseActivityEvent):
  def __init__(self, activity_dict):
    super(DropboxTlsActivityEvent, self).__init__(activity_dict,
                                                  data_type='java:rubanetra:dropbox_tls_activity')
    self.client_address = activity_dict.pop('clientAddress', None)
    self.server_address = activity_dict.pop('serverAddress', None)


class SpiderOakActivityEvent(BaseActivityEvent):
  def __init__(self, activity_dict):
    super(SpiderOakActivityEvent, self).__init__(activity_dict,
                                                 data_type='java:rubanetra:spideroak_activity')
    self.client_address = activity_dict.pop('clientAddress', None)
    self.server_address = activity_dict.pop('serverAddress', None)


class SkypePayloadActivityEvent(BaseActivityEvent):
  def __init__(self, activity_dict):
    super(SkypePayloadActivityEvent, self).__init__(activity_dict,
                                                    data_type='java:rubanetra:skype_payload_activity')
    self.source_object_id = activity_dict.pop('sourceObjectId', None)
    self.destination_object_id = activity_dict.pop('destinationObjectId', None)
    self.source_host = activity_dict.pop('sourceHost', None)
    self.destination_host = activity_dict.pop('destinationHost', None)


class JavaInstantEvent(TimestampEvent):
  """Convenience class for a Java Instant-based event."""

  def __init__(self, instant_epoch_seconds, instant_nano, usage, data_type='java:time:Instant'):
    """Initializes a Java instant-based event object.

    Args:
      java_java_instant_epoch_seconds: The Java epoch seconds value (long).
      java_java_instant_nano: The Java nano seconds value (long), will be reduced to microsecond precision.
      usage: The description of the usage of the instant value.
      data_type: The event data type. If not set data_type is derived
                 from DATA_TYPE.
    """
    super(JavaInstantEvent, self).__init__(
        timelib.Timestamp.FromPosixTimeWithMicrosecond(instant_epoch_seconds, instant_nano / 1000),
        usage, data_type)
    self.instant_epoch_seconds = instant_epoch_seconds
    self.instant_nano = instant_nano
    self.related_activity_uuids = list()

  @classmethod
  def from_java_instant_dict(cls, java_instant_as_dict, usage, data_type='java:time:Instant'):
    # TODO: validate fields
    instant_epoch_seconds = long(java_instant_as_dict.pop('epochSecond', -1))
    instant_nano = long(java_instant_as_dict.pop('nano', -1))

    return cls(instant_epoch_seconds, instant_nano, usage, data_type)

""" FIXME: This method is ineffective for now, because
    it is apparently not possible to specify a filter expression that
    is based on a boolean value.
"""
def string_to_boolean(s):
  """ Returns true, iff s.lower() in ('true', '1')

  :param s: a String representation of a boolean value
  :return:true, iff s.lower() in ('true', '1'), false otherwise
  """
  #return s.lower() in ('true', '1')
  return s


def merge_list_of_dicts_to_dict(list_of_dicts):
  """ Takes a list of dictionaries and transforms it to a flat dictionary, overwriting duplicate keys in the process.

  :param list_of_dicts: a list of dictionaries
  :return: a flat dictionary containing the keys and values of all dictionaries that were previously located inside the
           list. If two dictionaries contained the same key, the mapping of the last dictionary that contained that key
           will be included, while the older value is discarded.
  """
  if list_of_dicts is None or not isinstance(list_of_dicts, list) or isinstance(list_of_dicts, dict):
    return list_of_dicts

  return {k: v for d in list_of_dicts for k, v in d.items()}

# A dictionary of 'activityType' to class mappings.
activity_type_to_class_dict = {
    'ArpActivity': ArpActivityEvent,
    'DhcpActivity': DhcpActivityEvent,
    'DnsActivity': DnsActivityEvent,
    'EthernetActivity': EthernetActivityEvent,
    'FtpActivity': FtpActivityEvent,
    'HttpImageActivity': HttpImageActivityEvent,
    'HttpRequestActivity': HttpRequestActivityEvent,
    'HttpResponseActivity': HttpResponseActivityEvent,
    'Icmpv4Activity': Icmpv4ActivityEvent,
    'Icmpv6Activity': Icmpv6ActivityEvent,
    'IpActivity': IpActivityEvent,
    'Ipv4Activity': Ipv4ActivityEvent,
    'Ipv6Activity': Ipv6ActivityEvent,
    'MsnActivity': MsnActivityEvent,
    'NetbiosActivity': NetbiosActivityEvent,
    'PcapActivity': PcapActivityEvent,
    'Pop3Activity': Pop3ActivityEvent,
    'SmtpCommandActivity': SmtpCommandActivityEvent,
    'SmtpReplyActivity': SmtpReplyActivityEvent,
    'SmtpSendActivity': SmtpSendActivityEvent,
    'TcpActivity': TcpActivityEvent,
    'TelnetActivity': TelnetActivityEvent,
    'TlsActivity': TlsActivityEvent,
    'UdpActivity': UdpActivityEvent,
    'OpenSSHActivity': OpenSSHActivityEvent,
    'DropboxTlsActivity': DropboxTlsActivityEvent,
    'SpiderOakActivity': SpiderOakActivityEvent,
    'SkypePayloadActivity': SkypePayloadActivityEvent}

manager.ParsersManager.RegisterParser(RubanetraXmlParser)