#!/usr/bin/python # -*- coding: utf-8 -*- # # Copyright 2014 The Plaso Project Authors. # Please see the AUTHORS file for details on individual authors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import os import logging from plaso.lib import errors, event, timelib from plaso.events.time_events import TimestampEvent from plaso.lib.eventdata import EventTimestamp from plaso.parsers import interface from plaso.parsers import manager try: import xml.etree.cElementTree as ElementTree except ImportError: import xml.etree.ElementTree as ElementTree __author__ = 'Stefan Swerk (stefan_rubanetra@swerk.priv.at)' class RubanetraXmlParser(interface.BaseParser): """ This class represents the python parser-component of the Rubanetra project. Currently, it is only capable of parsing files adhering to the XML standard and depends on the 'xml.etree' library. """ NAME = 'rubanetra_xml_parser' DESCRIPTION = u'Rubanetra XML file parser' VERSION = u'0.0.6' RUBANETRA_METADATA_FIELDS = frozenset(['implementationVersion', 'implementationTitle', 'implementationVendor']) def Parse(self, parser_context, file_entry, parser_chain=None): """ Parses a XML file containing Rubanetra produced content. :param parser_context: A parser context object (instance of ParserContext). :param file_entry: A file entry object (instance of dfvfs.FileEntry). :param parser_chain: Optional string containing the parsing chain up to this point. The default is None. """ rubanetra_metadata_fields = set(self.RUBANETRA_METADATA_FIELDS) rubanetra_metadata_dict = dict() file_handle = None try: if file_entry is not None and file_entry.IsFile(): # Open the file read-only. file_handle = file_entry.GetFileObject() else: raise errors.UnableToParseFile(u'Not a valid Rubanetra file.') file_size = file_handle.get_size() if file_size <= 0: raise errors.UnableToParseFile(u'File size: {0:d} bytes is less or equal than 0.'.format(file_size)) # read from the beginning and check whether this is a XML file file_handle.seek(0, os.SEEK_SET) # get an iterable xml parser context xml_parser_context = ElementTree.iterparse(file_handle, events=('start', 'end')) # turn it into an iterator xml_parser_context = iter(xml_parser_context) # check whether this is a valid XML file try: xml_parser_context.next() except ElementTree.ParseError: raise errors.UnableToParseFile(u'Not a valid Rubanetra file (not XML).') # Add ourselves to the parser chain, which will be used in all subsequent # event creation in this parser. parser_chain = self._BuildParserChain(parser_chain) # read from the beginning to process metadata file_handle.seek(0, os.SEEK_SET) # get an iterable xml parser context xml_parser_context = ElementTree.iterparse(file_handle, events=('start', 'end')) # turn it into an iterator xml_parser_context = iter(xml_parser_context) # get the root element xml_event, xml_root = xml_parser_context.next() for xml_event, xml_elem in xml_parser_context: if xml_event == 'end': # ... process metadata ... if xml_elem.tag in rubanetra_metadata_fields: rubanetra_metadata_dict[xml_elem.tag] = xml_elem.text rubanetra_metadata_fields.discard(xml_elem.tag) elif len(rubanetra_metadata_fields) == 0: xml_elem.clear() xml_root.clear() break xml_elem.clear() xml_root.clear() if len(rubanetra_metadata_fields) != 0: raise errors.UnableToParseFile( u'Unable to verify metadata, required fields {0:s} could not be parsed'.format(rubanetra_metadata_fields)) self.validate_metadata(rubanetra_metadata_dict) # reopen the file handle file_handle.seek(0, os.SEEK_SET) # get an iterable xml parser context xml_parser_context = ElementTree.iterparse(file_handle, events=('start', 'end')) # turn it into an iterator xml_parser_context = iter(xml_parser_context) # get the root element xml_event, xml_root = xml_parser_context.next() for xml_event, xml_elem in xml_parser_context: if xml_event == 'end': # ... process activities ... if xml_elem.tag == 'activity': activity_dict_list = self.element_to_dict(xml_elem).pop('activity') event_objects = self.parse_activity(merge_list_of_dicts_to_dict(activity_dict_list)) parser_context.ProduceEvents(event_objects, parser_chain=parser_chain, file_entry=file_entry) xml_root.clear() finally: if file_handle is not None: file_handle.close() def parse_activity(self, activity_dict_merged): """ Takes a dictionary resembling an arbitrary activity and parses all fields in a recursive manner in order to process nested activities as well as ordinary leaf values. The following cases are currently handled: - Nested and split activities may occur as a list of dictionaries, as long as each activity dict contains an 'activityType' - Nested activities are allowed to occur under arbitrary key values - EventObjects are constructed using reflective access to the event class, i.e. each occurring 'activityType' must be mapped via 'activity_type_to_class_dict' and the mapped class must have a constructor that can handle the parsing process of a single flat activity using a dict. - If there is no mapping of an occurring 'activityType' within 'activity_type_to_class_dict', a BaseActivityEvent will be constructed instead. - Leaf values that do not represent an entire Activity will not be modified but instead passed as constructor argument to the corresponding EventObject implementation. :param activity_dict_merged: an arbitrary activity as dictionary, conforming to a certain 'activityType'. :return: a list of EventObjects that could be parsed from the given dictionary, corresponding to the value of 'activityType', if possible. Otherwise, either an empty list, in case the given dict does not contain a valid activity, or None, in case the given argument is not a dict, will be returned. """ if activity_dict_merged is None: return None event_objects = list() if isinstance(activity_dict_merged, list): for v in activity_dict_merged: if isinstance(v, dict) and 'activityType' in v: if len(v) == 1: # It is safe to assume that this is a nested and split activity. Therefore break the loop # and parse it. This assumption does no longer hold true in case of a scenario where # an activity_dict_list contains both, a split activity and another nested activity_dict. # Currently the xml parser handles this case by wrapping the nested activity_dict in another # list. return self.parse_activity(merge_list_of_dicts_to_dict(activity_dict_merged)) else: # this is a nested activity, parse and save objects child_evt_objs = self.parse_activity(v) if child_evt_objs is not None: for e in child_evt_objs: if isinstance(e, event.EventObject): event_objects += [e] else: event_objects += self.parse_activity(v) return event_objects # is it a leaf value or an actual activity? if not isinstance(activity_dict_merged, dict): return None # it is at least a dict, however, is it an activity or a value? activity_type = activity_dict_merged.get('activityType', None) if activity_type is not None: # it is an activity event_object_class = activity_type_to_class_dict.get(activity_type, BaseActivityEvent) event_objects = [event_object_class(activity_dict_merged)] # TODO check whether it is an actual class # everything that remains may be another activity or an unconsumed leaf value for k, v in activity_dict_merged.items(): child_evt_objs = self.parse_activity(v) # currently, the key attribute is not used -> TODO: find a way to link the child-events to its parent? if child_evt_objs is not None: for e in child_evt_objs: if isinstance(e, event.EventObject): event_objects += [e] return event_objects def parse_timestamp_events(self, activity_dict): """ Takes a dictionary of dictionaries that must contain at least two keys: - 'startInstant', a dictionary that corresponds to a serialized Java Instant object - 'endInstant', as above This method will produce a dictionary consisting of either one or two JavaInstantEvent objects. :param activity_dict: containing the 'startInstant' and 'endInstant' dictionaries :return: either one, iff 'startInstant' == 'endInstant', or two JavaInstantEvent-objects inside a dictionary. """ start_instant_dict = merge_list_of_dicts_to_dict(activity_dict.get('startInstant', None)) end_instant_dict = merge_list_of_dicts_to_dict(activity_dict.get('endInstant', None)) if start_instant_dict != end_instant_dict: instant_dict = dict(startInstant=None, endInstant=None) # interval start_instant_evt = JavaInstantEvent.from_java_instant_dict(start_instant_dict, EventTimestamp.FIRST_CONNECTED) end_instant_evt = JavaInstantEvent.from_java_instant_dict(end_instant_dict, EventTimestamp.LAST_CONNECTED) instant_dict['startInstant'] = start_instant_evt instant_dict['endInstant'] = end_instant_evt return instant_dict else: return dict(startInstant=JavaInstantEvent.from_java_instant_dict(start_instant_dict, u'Pcap time stamp')) def element_to_dict(self, elem): """ Internal method to transform a XML node to a dictionary. :param elem: the XML node :return: a dictionary containing the values below 'elem', using 'elem.tag' as respective keys """ return {elem.tag: map(self.element_to_dict, list(elem)) or elem.text} def validate_metadata(self, rubanetra_metadata_dict): """ Tries to verify that the parsed XML document corresponds to a known version to prevent potential issues due to version incompatibility. An exception will be raised if such a case is encountered. :param rubanetra_metadata_dict: a dictionary containing the basic Rubanetra metadata values """ if rubanetra_metadata_dict.get('implementationTitle') != u'Rubanetra': raise errors.UnableToParseFile(u'Unknown Rubanetra implementation title encountered.') version = rubanetra_metadata_dict.get('implementationVersion') if version != self.VERSION: logging.warning(u'Rubanetra version number mismatch, expected:{0:s}, actual:{1:s}'.format(self.VERSION, version)) def link_activity(self, event_object_from, event_object_to): """ This method is currently unused, however, in case it is necessary to group multiple events, a link between those events must be established. Whether a backtracking chain or a forward-chain should be established depends entirely on the caller. Currently, the UUID of 'event_object_to' will be appended to the list 'related_activity_uuids' of 'event_object_from'. """ if isinstance(event_object_to, BaseActivityEvent): event_object_from.related_activity_uuids.append(event_object_to.uuid) # TODO: else error class BaseActivityEvent(event.EventObject): def __init__(self, activity_dict, data_type='java:rubanetra:base_activity'): """Initializes the base event object. Args: activity_dict: A dictionary containing all related BaseActivity key/value pairs. """ super(BaseActivityEvent, self).__init__() if activity_dict is None: raise errors.UnableToParseFile self.data_type = data_type self.activity_type = activity_dict.pop('activityType', 'BaseActivity') self.description = activity_dict.pop('description', None) compound_frame_number_dict_list = activity_dict.pop('compoundFrameNumbers', None) self.compound_frame_number_list = list() if compound_frame_number_dict_list is not None: for d in compound_frame_number_dict_list: for k, v in d.items(): self.compound_frame_number_list.append(long(v)) # TODO checks self.optional_field_dict = merge_list_of_dicts_to_dict(activity_dict.pop('optionalFields', None)) self.replaced = string_to_boolean(activity_dict.pop('replaced', None)) self.source_address = activity_dict.pop('sourceAddressAsString', None) self.destination_address = activity_dict.pop('destinationAddressAsString', None) start_instant_dict = merge_list_of_dicts_to_dict(activity_dict.get('startInstant', None)) end_instant_dict = merge_list_of_dicts_to_dict(activity_dict.get('endInstant', None)) start_instant_evt = None if start_instant_dict != end_instant_dict: start_instant_evt = JavaInstantEvent.from_java_instant_dict(start_instant_dict, EventTimestamp.FIRST_CONNECTED) # interval end_instant_evt = JavaInstantEvent.from_java_instant_dict(end_instant_dict, EventTimestamp.LAST_CONNECTED) self.last_timestamp = end_instant_evt.timestamp if end_instant_evt is not None else None else: start_instant_evt = JavaInstantEvent.from_java_instant_dict(start_instant_dict, u'Pcap time stamp') self.timestamp = self.first_timestamp = start_instant_evt.timestamp if start_instant_evt is not None else None self.timestamp_desc = start_instant_evt.timestamp_desc self.related_activity_uuids = list() class PcapActivityEvent(BaseActivityEvent): def __init__(self, pcap_activity_dict): super(PcapActivityEvent, self).__init__(pcap_activity_dict, data_type='java:rubanetra:pcap_activity') pcap_packet = merge_list_of_dicts_to_dict(pcap_activity_dict.pop('pcapPacket', None)) if pcap_packet is not None: self.pcap_total_size = pcap_packet.pop('totalSize', None) self.pcap_frame_number = pcap_packet.pop('frameNumber', None) self.pcap_packet_wirelen = pcap_packet.pop('packetWirelen', None) self.pcap_header_count = pcap_packet.pop('headerCount', None) class HttpRequestActivityEvent(BaseActivityEvent): def __init__(self, http_request_activity_dict): super(HttpRequestActivityEvent, self).__init__(http_request_activity_dict, data_type='java:rubanetra:http_request_activity') self.http_version = http_request_activity_dict.pop('httpVersion', None) self.server_address = http_request_activity_dict.pop('serverAddress', None) self.client_address = http_request_activity_dict.pop('clientAddress', None) self.http_method = http_request_activity_dict.pop('httpMethod', None) self.http_query_string = http_request_activity_dict.pop('httpQueryString', None) self.http_query_parameters = http_request_activity_dict.pop('httpQueryParameters', None) self.http_request_header_dict = http_request_activity_dict.pop('requestHeaderMap', None) self.url = http_request_activity_dict.pop('url', None) http_request = merge_list_of_dicts_to_dict(http_request_activity_dict.pop('httpRequest', None)) if http_request is not None: self.orig_http_header = http_request.pop('header', None) self.content_type = http_request.pop('contentType', None) self.is_response = http_request.pop('response', None) self.jnetpcap_http_string = http_request.pop('JNetPcap-HTTP-String', None) self.source_address = self.client_address self.destination_address = self.server_address class HttpResponseActivityEvent(BaseActivityEvent): def __init__(self, http_response_activity_dict): super(HttpResponseActivityEvent, self).__init__(http_response_activity_dict, data_type='java:rubanetra:http_response_activity') self.http_version = http_response_activity_dict.pop('httpVersion', None) self.response_status_code = http_response_activity_dict.pop('responseStatusCode', None) self.response_status_line = http_response_activity_dict.pop('responseStatusLine', None) self.response_header_dict = http_response_activity_dict.pop('responseHeaderMap', None) http_response = merge_list_of_dicts_to_dict(http_response_activity_dict.pop('httpResponse', None)) if http_response is not None: self.orig_http_header = http_response.pop('header', None) self.content_type = http_response.pop('contentType', None) self.is_response = http_response.pop('response', None) self.jnetpcap_http_string = http_response.pop('JNetPcap-HTTP-String', None) class HttpImageActivityEvent(BaseActivityEvent): def __init__(self, http_image_activity_dict): super(HttpImageActivityEvent, self).__init__(http_image_activity_dict, data_type='java:rubanetra:http_image_activity') self.image_type = http_image_activity_dict.pop('imageType', None) self.image_path = http_image_activity_dict.pop('imagePath', None) class DnsActivityEvent(BaseActivityEvent): def __init__(self, dns_activity_dict): super(DnsActivityEvent, self).__init__(dns_activity_dict, data_type='java:rubanetra:dns_activity') self.question_record_list = dns_activity_dict.pop('questionRecords', None) self.answer_record_list = dns_activity_dict.pop('answerRecords', None) self.authority_record_list = dns_activity_dict.pop('authorityRecords', None) self.additional_record_list = dns_activity_dict.pop('additionalRecords', None) self.dns_message_header = merge_list_of_dicts_to_dict(dns_activity_dict.pop('dnsMessageHeader', None)) self.is_response_bool = string_to_boolean(dns_activity_dict.pop('response', None)) class ArpActivityEvent(BaseActivityEvent): def __init__(self, arp_activity_dict): super(ArpActivityEvent, self).__init__(arp_activity_dict, data_type='java:rubanetra:arp_activity') self.hardware_type = arp_activity_dict.pop('hardwareType', None) self.protocol_type = arp_activity_dict.pop('protocolType', None) self.hardware_address_length = arp_activity_dict.pop('hardwareAddressLength', None) self.protocol_address_length = arp_activity_dict.pop('protocolAddressLength', None) self.sender_mac_address = arp_activity_dict.pop('senderHardwareAddress', None) self.target_mac_address = arp_activity_dict.pop('targetHardwareAddress', None) self.sender_protocol_address = arp_activity_dict.pop('senderProtocolAddress', None) self.target_protocol_address = arp_activity_dict.pop('targetProtocolAddress', None) self.jnetpcap_arp = arp_activity_dict.pop('arp', None) class DhcpActivityEvent(BaseActivityEvent): def __init__(self, dhcp_activity_dict): super(DhcpActivityEvent, self).__init__(dhcp_activity_dict, data_type='java:rubanetra:dhcp_activity') self.dhcp_message = dhcp_activity_dict.pop('dhcpMessage', None) class EthernetActivityEvent(BaseActivityEvent): def __init__(self, ethernet_activity_dict): super(EthernetActivityEvent, self).__init__(ethernet_activity_dict, data_type='java:rubanetra:ethernet_activity') self.source_mac_address = ethernet_activity_dict.pop('sourceMacAddress', None) self.destination_mac_address = ethernet_activity_dict.pop('destinationMacAddress', None) self.ethernet_type = ethernet_activity_dict.pop('ethernetType', None) self.ethernet_type_enum = ethernet_activity_dict.pop('ethernetTypeEnum', None) self.jnetpcap_ethernet = ethernet_activity_dict.pop('ethernet', None) class FtpActivityEvent(BaseActivityEvent): def __init__(self, ftp_activity_dict): super(FtpActivityEvent, self).__init__(ftp_activity_dict, data_type='java:rubanetra:ftp_activity') self.ftp_type = ftp_activity_dict.pop('ftpActivityType', None) self.command = ftp_activity_dict.pop('command', None) self.reply = ftp_activity_dict.pop('reply', None) self.list = ftp_activity_dict.pop('list', None) class Icmpv4ActivityEvent(BaseActivityEvent): def __init__(self, icmpv4_activity_dict): super(Icmpv4ActivityEvent, self).__init__(icmpv4_activity_dict, data_type='java:rubanetra:icmpv4_activity') self.icmp_subtype = icmpv4_activity_dict.pop('icmpSubType', None) self.icmp_packet = icmpv4_activity_dict.pop('icmpPacket', None) self.icmp_message = icmpv4_activity_dict.pop('icmpMessage', None) self.icmp_type = icmpv4_activity_dict.pop('icmpType', None) self.icmp_code = icmpv4_activity_dict.pop('icmpCode', None) self.source_address = icmpv4_activity_dict.pop('sourceAddress', None) self.destination_address = icmpv4_activity_dict.pop('destinationAddress', None) self.identifier = icmpv4_activity_dict.pop('identifier', None) self.sequence = icmpv4_activity_dict.pop('sequence', None) self.jnetpcap_icmp = icmpv4_activity_dict.pop('icmp', None) class Icmpv6ActivityEvent(BaseActivityEvent): def __init__(self, icmpv6_activity_dict): super(Icmpv6ActivityEvent, self).__init__(icmpv6_activity_dict, data_type='java:rubanetra:icmpv6_activity') self.icmp_subtype = icmpv6_activity_dict.pop('icmpSubType', None) self.icmp_packet = icmpv6_activity_dict.pop('icmpPacket', None) self.icmp_message = icmpv6_activity_dict.pop('icmpMessage', None) self.icmp_type = icmpv6_activity_dict.pop('icmpType', None) self.jnetpcap_icmp = icmpv6_activity_dict.pop('icmp', None) class IpActivityEvent(BaseActivityEvent): def __init__(self, ip_activity_dict): super(IpActivityEvent, self).__init__(ip_activity_dict, data_type='java:rubanetra:ip_activity') self.version = ip_activity_dict.pop('version', None) self.protocol = ip_activity_dict.pop('protocol', None) self.source_address = ip_activity_dict.pop('sourceAddress', None) self.destination_address = ip_activity_dict.pop('destinationAddress', None) class Ipv4ActivityEvent(BaseActivityEvent): def __init__(self, ip_activity_dict): super(Ipv4ActivityEvent, self).__init__(ip_activity_dict, data_type='java:rubanetra:ipv4_activity') self.internet_header_length = ip_activity_dict.pop('internetHeaderLength', None) self.differentiated_services_code_point = ip_activity_dict.pop('differentiatedServicesCodePoint', None) self.total_length = ip_activity_dict.pop('totalLength', None) self.identification = ip_activity_dict.pop('identification', None) self.flags = ip_activity_dict.pop('flags', None) self.fragment_offset = ip_activity_dict.pop('fragmentOffset', None) self.time_to_live = ip_activity_dict.pop('timeToLive', None) self.header_checksum = ip_activity_dict.pop('headerChecksum', None) self.options = ip_activity_dict.pop('options', None) self.jnetpcap_ip4 = ip_activity_dict.pop('ipv4', None) class Ipv6ActivityEvent(BaseActivityEvent): def __init__(self, ip_activity_dict): super(Ipv6ActivityEvent, self).__init__(ip_activity_dict, data_type='java:rubanetra:ipv6_activity') self.traffic_class = ip_activity_dict.pop('trafficClass', None) self.flow_label = ip_activity_dict.pop('flowLabel', None) self.payload_length = ip_activity_dict.pop('payloadLength', None) self.next_header = ip_activity_dict.pop('nextHeader', None) self.hop_limit = ip_activity_dict.pop('hopLimit', None) self.jnetpcap_ip6 = ip_activity_dict.pop('ipv6', None) self.kraken_ip6 = ip_activity_dict.pop('ipv6Packet', None) class MsnActivityEvent(BaseActivityEvent): def __init__(self, activity_dict): super(MsnActivityEvent, self).__init__(activity_dict, data_type='java:rubanetra:msn_activity') self.account = activity_dict.pop('account', None) self.chat = activity_dict.pop('chat', None) class NetbiosActivityEvent(BaseActivityEvent): def __init__(self, activity_dict): super(NetbiosActivityEvent, self).__init__(activity_dict, data_type='java:rubanetra:netbios_activity') self.datagram_packet = activity_dict.pop('datagramPacket', None) self.name_packet = activity_dict.pop('namePacket', None) class Pop3ActivityEvent(BaseActivityEvent): def __init__(self, activity_dict): super(Pop3ActivityEvent, self).__init__(activity_dict, data_type='java:rubanetra:pop3_activity') self.sub_type = activity_dict.pop('subType', None) self.header = activity_dict.pop('header', None) self.data = activity_dict.pop('data', None) self.command = activity_dict.pop('command', None) self.response = activity_dict.pop('response', None) class SmtpCommandActivityEvent(BaseActivityEvent): def __init__(self, activity_dict): super(SmtpCommandActivityEvent, self).__init__(activity_dict, data_type='java:rubanetra:smtp_command_activity') self.command = activity_dict.pop('command', None) self.parameter = activity_dict.pop('parameter', None) class SmtpReplyActivityEvent(BaseActivityEvent): def __init__(self, activity_dict): super(SmtpReplyActivityEvent, self).__init__(activity_dict, data_type='java:rubanetra:smtp_reply_activity') self.code = activity_dict.pop('code', None) self.message = activity_dict.pop('message', None) class SmtpSendActivityEvent(BaseActivityEvent): def __init__(self, activity_dict): super(SmtpSendActivityEvent, self).__init__(activity_dict, data_type='java:rubanetra:smtp_send_activity') self.header = activity_dict.pop('header', None) self.data = activity_dict.pop('data', None) class Snmpv1ActivityEvent(BaseActivityEvent): def __init__(self, activity_dict): super(Snmpv1ActivityEvent, self).__init__(activity_dict, data_type='java:rubanetra:snmpv1_activity') self.pdu = activity_dict.pop('pdu', None) self.source_socket_address = activity_dict.pop('sourceSocketAddress', None) self.destination_socket_address = activity_dict.pop('destinationSocketAddress', None) class Snmpv2ActivityEvent(BaseActivityEvent): def __init__(self, activity_dict): super(Snmpv2ActivityEvent, self).__init__(activity_dict, data_type='java:rubanetra:snmpv2_activity') self.pdu = activity_dict.pop('pdu', None) self.source_socket_address = activity_dict.pop('sourceSocketAddress', None) self.destination_socket_address = activity_dict.pop('destinationSocketAddress', None) class TcpActivityEvent(BaseActivityEvent): def __init__(self, activity_dict): super(TcpActivityEvent, self).__init__(activity_dict, data_type='java:rubanetra:tcp_activity') self.source_port = activity_dict.pop('sourcePort', None) self.destination_port = activity_dict.pop('destinationPort', None) self.sequence_number = activity_dict.pop('sequenceNumber', None) self.acknowledge_number = activity_dict.pop('acknowledgeNumber', None) self.relative_sequence_number = activity_dict.pop('relativeSequenceNumber', None) self.relative_acknowledge_number = activity_dict.pop('relativeAcknowledgeNumber', None) self.data_offset = activity_dict.pop('dataOffset', None) self.control_bits = activity_dict.pop('controlBits', None) self.window_size = activity_dict.pop('windowSize', None) self.checksum = activity_dict.pop('checksum', None) self.urgent_pointer = activity_dict.pop('urgentPointer', None) self.tcp_length = activity_dict.pop('tcpLength', None) self.options = activity_dict.pop('options', None) self.padding = activity_dict.pop('padding', None) self.syn = activity_dict.pop('syn', None) self.ack = activity_dict.pop('ack', None) self.psh = activity_dict.pop('psh', None) self.fin = activity_dict.pop('fin', None) self.rst = activity_dict.pop('rst', None) self.urg = activity_dict.pop('urg', None) self.direction = activity_dict.pop('direction', None) self.client_state = activity_dict.pop('clientState', None) self.server_state = activity_dict.pop('serverState', None) self.jnetpcap_tcp = activity_dict.pop('tcp', None) self.source_address = activity_dict.pop('sourceAddress', None) self.destination_address = activity_dict.pop('destinationAddress', None) self.source_socket_address = activity_dict.pop('sourceSocketAddress', None) self.destination_socket_address = activity_dict.pop('destinationSocketAddress', None) class TelnetActivityEvent(BaseActivityEvent): def __init__(self, activity_dict): super(TelnetActivityEvent, self).__init__(activity_dict, data_type='java:rubanetra:telnet_activity') self.sub_type = activity_dict.pop('subType', None) self.command = activity_dict.pop('command', None) self.option = activity_dict.pop('option', None) self.ansi_mode = activity_dict.pop('ansiMode', None) self.arguments = activity_dict.pop('arguments', None) self.text = activity_dict.pop('text', None) self.title = activity_dict.pop('title', None) class TlsActivityEvent(BaseActivityEvent): def __init__(self, activity_dict): super(TlsActivityEvent, self).__init__(activity_dict, data_type='java:rubanetra:tls_activity') self.client_to_server_traffic = activity_dict.pop('clientToServerTraffic', None) self.server_to_client_traffic = activity_dict.pop('serverToClientTraffic', None) class UdpActivityEvent(BaseActivityEvent): def __init__(self, activity_dict): super(UdpActivityEvent, self).__init__(activity_dict, data_type='java:rubanetra:udp_activity') self.source_port = activity_dict.pop('sourcePort', None) self.destination_port = activity_dict.pop('destinationPort', None) self.length = activity_dict.pop('length', None) self.checksum = activity_dict.pop('checksum', None) self.jnetpcap_udp = activity_dict.pop('udp', None) self.source_socket_address = activity_dict.pop('sourceSocketAddress', None) self.destination_socket_address = activity_dict.pop('destinationSocketAddress', None) class OpenSSHActivityEvent(BaseActivityEvent): def __init__(self, activity_dict): super(OpenSSHActivityEvent, self).__init__(activity_dict, data_type='java:rubanetra:open_ssh_activity') self.client_to_server_traffic = activity_dict.pop('clientToServerTraffic', None) self.server_to_client_traffic = activity_dict.pop('serverToClientTraffic', None) class DropboxTlsActivityEvent(BaseActivityEvent): def __init__(self, activity_dict): super(DropboxTlsActivityEvent, self).__init__(activity_dict, data_type='java:rubanetra:dropbox_tls_activity') self.client_address = activity_dict.pop('clientAddress', None) self.server_address = activity_dict.pop('serverAddress', None) class SpiderOakActivityEvent(BaseActivityEvent): def __init__(self, activity_dict): super(SpiderOakActivityEvent, self).__init__(activity_dict, data_type='java:rubanetra:spideroak_activity') self.client_address = activity_dict.pop('clientAddress', None) self.server_address = activity_dict.pop('serverAddress', None) class SkypePayloadActivityEvent(BaseActivityEvent): def __init__(self, activity_dict): super(SkypePayloadActivityEvent, self).__init__(activity_dict, data_type='java:rubanetra:skype_payload_activity') self.source_object_id = activity_dict.pop('sourceObjectId', None) self.destination_object_id = activity_dict.pop('destinationObjectId', None) self.source_host = activity_dict.pop('sourceHost', None) self.destination_host = activity_dict.pop('destinationHost', None) class JavaInstantEvent(TimestampEvent): """Convenience class for a Java Instant-based event.""" def __init__(self, instant_epoch_seconds, instant_nano, usage, data_type='java:time:Instant'): """Initializes a Java instant-based event object. Args: java_java_instant_epoch_seconds: The Java epoch seconds value (long). java_java_instant_nano: The Java nano seconds value (long), will be reduced to microsecond precision. usage: The description of the usage of the instant value. data_type: The event data type. If not set data_type is derived from DATA_TYPE. """ super(JavaInstantEvent, self).__init__( timelib.Timestamp.FromPosixTimeWithMicrosecond(instant_epoch_seconds, instant_nano / 1000), usage, data_type) self.instant_epoch_seconds = instant_epoch_seconds self.instant_nano = instant_nano self.related_activity_uuids = list() @classmethod def from_java_instant_dict(cls, java_instant_as_dict, usage, data_type='java:time:Instant'): # TODO: validate fields instant_epoch_seconds = long(java_instant_as_dict.pop('epochSecond', -1)) instant_nano = long(java_instant_as_dict.pop('nano', -1)) return cls(instant_epoch_seconds, instant_nano, usage, data_type) """ FIXME: This method is ineffective for now, because it is apparently not possible to specify a filter expression that is based on a boolean value. """ def string_to_boolean(s): """ Returns true, iff s.lower() in ('true', '1') :param s: a String representation of a boolean value :return:true, iff s.lower() in ('true', '1'), false otherwise """ #return s.lower() in ('true', '1') return s def merge_list_of_dicts_to_dict(list_of_dicts): """ Takes a list of dictionaries and transforms it to a flat dictionary, overwriting duplicate keys in the process. :param list_of_dicts: a list of dictionaries :return: a flat dictionary containing the keys and values of all dictionaries that were previously located inside the list. If two dictionaries contained the same key, the mapping of the last dictionary that contained that key will be included, while the older value is discarded. """ if list_of_dicts is None or not isinstance(list_of_dicts, list) or isinstance(list_of_dicts, dict): return list_of_dicts return {k: v for d in list_of_dicts for k, v in d.items()} # A dictionary of 'activityType' to class mappings. activity_type_to_class_dict = { 'ArpActivity': ArpActivityEvent, 'DhcpActivity': DhcpActivityEvent, 'DnsActivity': DnsActivityEvent, 'EthernetActivity': EthernetActivityEvent, 'FtpActivity': FtpActivityEvent, 'HttpImageActivity': HttpImageActivityEvent, 'HttpRequestActivity': HttpRequestActivityEvent, 'HttpResponseActivity': HttpResponseActivityEvent, 'Icmpv4Activity': Icmpv4ActivityEvent, 'Icmpv6Activity': Icmpv6ActivityEvent, 'IpActivity': IpActivityEvent, 'Ipv4Activity': Ipv4ActivityEvent, 'Ipv6Activity': Ipv6ActivityEvent, 'MsnActivity': MsnActivityEvent, 'NetbiosActivity': NetbiosActivityEvent, 'PcapActivity': PcapActivityEvent, 'Pop3Activity': Pop3ActivityEvent, 'SmtpCommandActivity': SmtpCommandActivityEvent, 'SmtpReplyActivity': SmtpReplyActivityEvent, 'SmtpSendActivity': SmtpSendActivityEvent, 'TcpActivity': TcpActivityEvent, 'TelnetActivity': TelnetActivityEvent, 'TlsActivity': TlsActivityEvent, 'UdpActivity': UdpActivityEvent, 'OpenSSHActivity': OpenSSHActivityEvent, 'DropboxTlsActivity': DropboxTlsActivityEvent, 'SpiderOakActivity': SpiderOakActivityEvent, 'SkypePayloadActivity': SkypePayloadActivityEvent} manager.ParsersManager.RegisterParser(RubanetraXmlParser)