Import from old repository

This commit is contained in:
Stefan
2020-04-06 18:48:34 +02:00
commit 0da6783a45
762 changed files with 103065 additions and 0 deletions
+56
View File
@@ -0,0 +1,56 @@
#!/usr/bin/python
# -*- coding: utf-8 -*-
# Copyright 2013 The Plaso Project Authors.
# Please see the AUTHORS file for details on individual authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""This file contains an import statement for each filter."""
import logging
from plaso.filters import dynamic_filter
from plaso.filters import eventfilter
from plaso.filters import filterlist
from plaso.lib import filter_interface
from plaso.lib import errors
def ListFilters():
"""Generate a list of all available filters."""
filters = []
for cl in filter_interface.FilterObject.classes:
filters.append(filter_interface.FilterObject.classes[cl]())
return filters
def GetFilter(filter_string):
"""Returns the first filter that matches the filter string.
Args:
filter_string: A filter string for any of the available filters.
Returns:
The first FilterObject found matching the filter string. If no FilterObject
is available for this filter string None is returned.
"""
if not filter_string:
return
for filter_obj in ListFilters():
try:
filter_obj.CompileFilter(filter_string)
return filter_obj
except errors.WrongPlugin:
logging.debug(u'Filterstring [{}] is not a filter: {}'.format(
filter_string, filter_obj.filter_name))
+162
View File
@@ -0,0 +1,162 @@
#!/usr/bin/python
# -*- coding: utf-8 -*-
# Copyright 2013 The Plaso Project Authors.
# Please see the AUTHORS file for details on individual authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""This file contains definition for a selective fields EventObjectFilter."""
from plaso.lib import errors
from plaso.lib import lexer
from plaso.filters import eventfilter
class SelectiveLexer(lexer.Lexer):
"""A simple selective filter lexer implementation."""
tokens = [
lexer.Token('INITIAL', r'SELECT', '', 'FIELDS'),
lexer.Token('FIELDS', r'(.+) WHERE ', 'SetFields', 'FILTER'),
lexer.Token('FIELDS', r'(.+) LIMIT', 'SetFields', 'LIMIT_END'),
lexer.Token('FIELDS', r'(.+) SEPARATED BY', 'SetFields', 'SEPARATE'),
lexer.Token('FIELDS', r'(.+)$', 'SetFields', 'END'),
lexer.Token('FILTER', r'(.+) SEPARATED BY', 'SetFilter', 'SEPARATE'),
lexer.Token('FILTER', r'(.+) LIMIT', 'SetFilter', 'LIMIT_END'),
lexer.Token('FILTER', r'(.+)$', 'SetFilter', 'END'),
lexer.Token('SEPARATE', r' ', '', ''), # Ignore white space here.
lexer.Token('SEPARATE', r'LIMIT', '', 'LIMIT_END'),
lexer.Token(
'SEPARATE', r'[\'"]([^ \'"]+)[\'"] LIMIT', 'SetSeparator',
'LIMIT_END'),
lexer.Token(
'SEPARATE', r'[\'"]([^ \'"]+)[\'"]$', 'SetSeparator', 'END'),
lexer.Token(
'SEPARATE', r'(.+)$', 'SetSeparator', 'END'),
lexer.Token(
'LIMIT_END', r'SEPARATED BY [\'"]([^\'"]+)[\'"]', 'SetSeparator', ''),
lexer.Token('LIMIT_END', r'(.+) SEPARATED BY', 'SetLimit', 'SEPARATE'),
lexer.Token('LIMIT_END', r'(.+)$', 'SetLimit', 'END')]
def __init__(self, data=''):
"""Initialize the lexer."""
self.fields = []
self.limit = 0
self.lex_filter = None
self.separator = u','
super(SelectiveLexer, self).__init__(data)
def SetFilter(self, match, **_):
"""Set the filter query."""
filter_match = match.group(1)
if 'LIMIT' in filter_match:
# This only occurs in the case where we have "LIMIT X SEPARATED BY".
self.lex_filter, _, push_back = filter_match.rpartition('LIMIT')
self.PushBack('LIMIT {} SEPARATED BY '.format(push_back))
else:
self.lex_filter = filter_match
def SetSeparator(self, match, **_):
"""Set the separator of the output, only uses the first char."""
separator = match.group(1)
if separator:
self.separator = separator[0]
def SetLimit(self, match, **_):
"""Set the row limit."""
try:
limit = int(match.group(1))
except ValueError:
self.Error('Invalid limit value, should be int [{}] = {}'.format(
type(match.group(1)), match.group(1)))
limit = 0
self.limit = limit
def SetFields(self, match, **_):
"""Set the selective fields."""
text = match.group(1).lower()
field_text, _, _ = text.partition(' from ')
use_field_text = field_text.replace(' ', '')
if ',' in use_field_text:
self.fields = use_field_text.split(',')
else:
self.fields = [use_field_text]
class DynamicFilter(eventfilter.EventObjectFilter):
"""A twist to the EventObjectFilter allowing output fields to be selected.
This filter is essentially the same as the EventObjectFilter except it wraps
it in a selection of which fields should be included by an output module that
has support for selective fields. That is to say the filter:
SELECT field_a, field_b WHERE attribute contains 'text'
Will use the EventObjectFilter "attribute contains 'text'" and at the same
time indicate to the appropriate output module that the user wants only the
fields field_a and field_b to be used in the output.
"""
@property
def fields(self):
"""Set the fields property."""
return self._fields
@property
def limit(self):
"""Return the limit of row counts."""
return self._limit
@property
def separator(self):
"""Return the separator value."""
return self._separator
def __init__(self):
"""Initialize the selective EventObjectFilter."""
super(DynamicFilter, self).__init__()
self._fields = []
self._limit = 0
self._separator = u','
def CompileFilter(self, filter_string):
"""Compile the filter string into a EventObjectFilter matcher."""
lex = SelectiveLexer(filter_string)
_ = lex.NextToken()
if lex.error:
raise errors.WrongPlugin('Malformed filter string.')
_ = lex.NextToken()
if lex.error:
raise errors.WrongPlugin('No fields defined.')
if lex.state is not 'END':
while lex.state is not 'END':
_ = lex.NextToken()
if lex.error:
raise errors.WrongPlugin('No filter defined for DynamicFilter.')
if lex.state != 'END':
raise errors.WrongPlugin(
'Malformed DynamicFilter, end state not reached.')
self._fields = lex.fields
self._limit = lex.limit
self._separator = unicode(lex.separator)
if lex.lex_filter:
super(DynamicFilter, self).CompileFilter(lex.lex_filter)
else:
self.matcher = None
+85
View File
@@ -0,0 +1,85 @@
#!/usr/bin/python
# -*- coding: utf-8 -*-
# Copyright 2013 The Plaso Project Authors.
# Please see the AUTHORS file for details on individual authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Tests for the DynamicFilter filter."""
import unittest
from plaso.filters import dynamic_filter
from plaso.filters import test_helper
class DynamicFilterTest(test_helper.FilterTestHelper):
"""Tests for the DynamicFilter filter."""
def setUp(self):
"""Sets up the needed objects used throughout the test."""
self.test_filter = dynamic_filter.DynamicFilter()
def testFilterFail(self):
"""Run few tests that should not be a proper filter."""
self.TestFail('/tmp/file_that_most_likely_does_not_exist')
self.TestFail('some random stuff that is destined to fail')
self.TestFail('some_stuff is "random" and other_stuff ')
self.TestFail('some_stuff is "random" and other_stuff is not "random"')
self.TestFail('SELECT stuff FROM machine WHERE conditions are met')
self.TestFail('SELECT field_a, field_b WHERE ')
self.TestFail('SELECT field_a, field_b SEPARATED BY')
self.TestFail('SELECT field_a, SEPARATED BY field_b WHERE ')
self.TestFail('SELECT field_a, field_b LIMIT WHERE')
def testFilterApprove(self):
self.TestTrue('SELECT stuff FROM machine WHERE some_stuff is "random"')
self.TestTrue('SELECT field_a, field_b, field_c')
self.TestTrue('SELECT field_a, field_b, field_c SEPARATED BY "%"')
self.TestTrue('SELECT field_a, field_b, field_c LIMIT 10')
self.TestTrue('SELECT field_a, field_b, field_c LIMIT 10 SEPARATED BY "|"')
self.TestTrue('SELECT field_a, field_b, field_c SEPARATED BY "|" LIMIT 10')
self.TestTrue('SELECT field_a, field_b, field_c WHERE date > "2012"')
self.TestTrue(
'SELECT field_a, field_b, field_c WHERE date > "2012" LIMIT 100')
self.TestTrue((
'SELECT field_a, field_b, field_c WHERE date > "2012" SEPARATED BY "@"'
' LIMIT 100'))
self.TestTrue((
'SELECT parser, date, time WHERE some_stuff is "random" and '
'date < "2021-02-14 14:51:23"'))
def testFilterFields(self):
query = 'SELECT stuff FROM machine WHERE some_stuff is "random"'
self.test_filter.CompileFilter(query)
self.assertEquals(['stuff'], self.test_filter.fields)
query = 'SELECT stuff, a, b, date FROM machine WHERE some_stuff is "random"'
self.test_filter.CompileFilter(query)
self.assertEquals(['stuff', 'a', 'b', 'date'], self.test_filter.fields)
query = 'SELECT date, message, zone, hostname WHERE some_stuff is "random"'
self.test_filter.CompileFilter(query)
self.assertEquals(['date', 'message', 'zone', 'hostname'],
self.test_filter.fields)
query = 'SELECT hlutir'
self.test_filter.CompileFilter(query)
self.assertEquals(['hlutir'], self.test_filter.fields)
query = 'SELECT hlutir LIMIT 10'
self.test_filter.CompileFilter(query)
self.assertEquals(['hlutir'], self.test_filter.fields)
self.assertEquals(10, self.test_filter.limit)
if __name__ == '__main__':
unittest.main()
+40
View File
@@ -0,0 +1,40 @@
#!/usr/bin/python
# -*- coding: utf-8 -*-
# Copyright 2013 The Plaso Project Authors.
# Please see the AUTHORS file for details on individual authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""This file contains definition for a simple filter."""
from plaso.lib import errors
from plaso.lib import filter_interface
from plaso.lib import pfilter
class EventObjectFilter(filter_interface.FilterObject):
"""A simple filter using the objectfilter library."""
def CompileFilter(self, filter_string):
"""Compile the filter string into a filter matcher."""
self.matcher = pfilter.GetMatcher(filter_string, True)
if not self.matcher:
raise errors.WrongPlugin('Malformed filter string.')
def Match(self, event_object):
"""Evaluate an EventObject against a filter."""
if not self.matcher:
return True
self._decision = self.matcher.Matches(event_object)
return self._decision
+43
View File
@@ -0,0 +1,43 @@
#!/usr/bin/python
# -*- coding: utf-8 -*-
# Copyright 2013 The Plaso Project Authors.
# Please see the AUTHORS file for details on individual authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Tests for the EventObjectFilter filter."""
import unittest
from plaso.filters import test_helper
from plaso.filters import eventfilter
class EventObjectFilterTest(test_helper.FilterTestHelper):
"""Tests for the EventObjectFilter filter."""
def setUp(self):
"""Sets up the needed objects used throughout the test."""
self.test_filter = eventfilter.EventObjectFilter()
def testFilterFail(self):
"""Run few tests that should not be a proper filter."""
self.TestFail('SELECT stuff FROM machine WHERE conditions are met')
self.TestFail('/tmp/file_that_most_likely_does_not_exist')
self.TestFail('some random stuff that is destined to fail')
self.TestFail('some_stuff is "random" and other_stuff ')
def testFilterApprove(self):
self.TestTrue('some_stuff is "random" and other_stuff is not "random"')
if __name__ == '__main__':
unittest.main()
+109
View File
@@ -0,0 +1,109 @@
#!/usr/bin/python
# -*- coding: utf-8 -*-
# Copyright 2013 The Plaso Project Authors.
# Please see the AUTHORS file for details on individual authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""This file contains definition for a list of ObjectFilter."""
import os
import yaml
import logging
from plaso.lib import errors
from plaso.lib import filter_interface
from plaso.lib import pfilter
def IncludeKeyword(loader, node):
"""A constructor for the include keyword in YAML."""
filename = loader.construct_scalar(node)
if os.path.isfile(filename):
with open(filename, 'rb') as fh:
try:
data = yaml.safe_load(fh)
except yaml.ParserError as exception:
logging.error(u'Unable to load rule file with error: {0:s}'.format(
exception))
return None
return data
class ObjectFilterList(filter_interface.FilterObject):
"""A series of Pfilter filters along with metadata."""
def CompileFilter(self, filter_string):
"""Compile a set of ObjectFilters defined in an YAML file."""
if not os.path.isfile(filter_string):
raise errors.WrongPlugin((
'ObjectFilterList requires an YAML file to be passed on, this filter '
'string is not a file.'))
yaml.add_constructor('!include', IncludeKeyword,
Loader=yaml.loader.SafeLoader)
results = None
with open(filter_string, 'rb') as fh:
try:
results = yaml.safe_load(fh)
except (yaml.scanner.ScannerError, IOError) as exception:
raise errors.WrongPlugin(
u'Unable to parse YAML file with error: {0:s}.'.format(exception))
self.filters = []
if type(results) is dict:
self._ParseEntry(results)
elif type(results) is list:
for result in results:
if type(result) is not dict:
raise errors.WrongPlugin(
u'Wrong format of YAML file, entry not a dict ({})'.format(
type(result)))
self._ParseEntry(result)
else:
raise errors.WrongPlugin(
u'Wrong format of YAML file, entry not a dict ({})'.format(
type(result)))
def _ParseEntry(self, entry):
"""Parse a single YAML filter entry."""
# A single file with a list of filters to parse.
for name, meta in entry.items():
if 'filter' not in meta:
raise errors.WrongPlugin(
u'Entry inside {} does not contain a filter statement.'.format(
name))
matcher = pfilter.GetMatcher(meta.get('filter'), True)
if not matcher:
raise errors.WrongPlugin(
u'Filter entry [{0:s}] malformed for rule: <{1:s}>'.format(
meta.get('filter'), name))
self.filters.append((name, matcher, meta))
def Match(self, event_object):
"""Evaluate an EventObject against a pfilter."""
if not self.filters:
return True
for name, matcher, meta in self.filters:
self._decision = matcher.Matches(event_object)
if self._decision:
self._reason = u'[{}] {} {}'.format(
name, meta.get('description', 'N/A'), u' - '.join(
meta.get('urls', [])))
return True
return False
+98
View File
@@ -0,0 +1,98 @@
#!/usr/bin/python
# -*- coding: utf-8 -*-
#
# Copyright 2013 The Plaso Project Authors.
# Please see the AUTHORS file for details on individual authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Tests for the PFilters filter."""
import os
import logging
import tempfile
import unittest
from plaso.filters import filterlist
from plaso.filters import test_helper
class ObjectFilterTest(test_helper.FilterTestHelper):
"""Tests for the ObjectFilterList filter."""
def setUp(self):
"""Sets up the needed objects used throughout the test."""
self.test_filter = filterlist.ObjectFilterList()
def testFilterFail(self):
"""Run few tests that should not be a proper filter."""
self.TestFail('SELECT stuff FROM machine WHERE conditions are met')
self.TestFail('/tmp/file_that_most_likely_does_not_exist')
self.TestFail('some random stuff that is destined to fail')
self.TestFail('some_stuff is "random" and other_stuff ')
self.TestFail('some_stuff is "random" and other_stuff is not "random"')
def CreateFileAndTest(self, content):
"""Creates a file and then runs the test."""
name = ''
with tempfile.NamedTemporaryFile(delete=False) as file_object:
name = file_object.name
file_object.write(content)
self.TestTrue(name)
try:
os.remove(name)
except (OSError, IOError) as exception:
logging.warning(
u'Unable to remove temporary file: {0:s} with error: {1:s}'.format(
name, exception))
def testFilterApprove(self):
one_rule = u'\n'.join([
u'Again_Dude:',
u' description: Heavy artillery caught on fire',
u' case_nr: 62345',
u' analysts: [anonymous]',
u' urls: [cnn.com,microsoft.com]',
u' filter: message contains "dude where is my car"'])
self.CreateFileAndTest(one_rule)
collection = u'\n'.join([
u'Rule_Dude:',
u' description: This is the very case I talk about, a lot',
u' case_nr: 1235',
u' analysts: [dude, jack, horn]',
u' urls: [mbl.is,visir.is]',
(u' filter: date > "2012-01-01 10:54:13" and parser not contains '
u'"evtx"'),
u'',
u'Again_Dude:',
u' description: Heavy artillery caught on fire',
u' case_nr: 62345',
u' analysts: [smith, perry, john]',
u' urls: [cnn.com,microsoft.com]',
u' filter: message contains "dude where is my car"',
u'',
u'Third_Rule_Of_Thumb:',
u' description: Another ticket for another day.',
u' case_nr: 234',
u' analysts: [joe]',
u' urls: [mbl.is,symantec.com/whereevillies,virustotal.com/myhash]',
u' filter: evil_bit is 1'])
self.CreateFileAndTest(collection)
if __name__ == '__main__':
unittest.main()
+50
View File
@@ -0,0 +1,50 @@
#!/usr/bin/python
# -*- coding: utf-8 -*-
# Copyright 2013 The Plaso Project Authors.
# Please see the AUTHORS file for details on individual authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""This file contains helper function and classes for filters."""
import unittest
from plaso.lib import errors
class FilterTestHelper(unittest.TestCase):
"""A simple class that provides helper functions for testing filters."""
def setUp(self):
"""This should be overwritten."""
self.test_filter = None
def TestTrue(self, query):
"""A quick test that should return a valid filter."""
if not self.test_filter:
self.assertTrue(False)
try:
self.test_filter.CompileFilter(query)
# And a success.
self.assertTrue(True)
except errors.WrongPlugin:
# Let the test fail.
self.assertTrue(False)
def TestFail(self, query):
"""A quick failure test with a filter."""
if not self.test_filter:
self.assertTrue(False)
with self.assertRaises(errors.WrongPlugin):
self.test_filter.CompileFilter(query)