mstoeck3/autopsy-flatpak
1
0
Fork 0
mirror of https://github.com/overcuriousity/autopsy-flatpak.git synced 2025-07-06 21:00:22 +00:00
Code Issues Packages Projects Releases Wiki Activity

now working to update update script and documentation for excel

Browse Source
This commit is contained in:
Greg DiCristofaro 2020-09-18 15:57:56 -04:00
parent 7424e25c37
commit 2ffdc1df09
7 changed files with 138 additions and 83 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

56
release_scripts/localization_scripts/allbundlesscript.py
View File

@ -3,44 +3,21 @@ This script requires the python libraries: gitpython and jproperties. As a cons
git >= 1.7.0 and python >= 3.4. This script relies on fetching 'HEAD' from current branch. So make sure
repo is on correct branch (i.e. develop).
"""
import collections
import sys
from envutil import get_proj_dir
from excelutil import records_to_excel
from fileutil import get_filename_addition, OMITTED_ADDITION
from excelutil import write_results_to_xlsx
from gitutil import get_property_file_entries, get_commit_id, get_git_root
from csvutil import records_to_csv
from typing import Union, TypedDict, List
from csvutil import write_results_to_csv
from typing import Union
import re
import argparse
class ProcessingResult:
results: List[List[str]]
omitted: Union[List[List[str]], None]
def __init__(self, results: List[List[str]], omitted: Union[List[List[str]], None]):
self.results = results
self.omitted = omitted
def write_items_to_csv(results: ProcessingResult, output_path: str):
records_to_csv(output_path, results.results)
if results.omitted:
records_to_csv(get_filename_addition(output_path, OMITTED_ADDITION), results.omitted)
def write_items_to_xlsx(results: ProcessingResult, output_path: str):
workbook = collections.OrderedDict([('results', results.results)])
if results.omitted:
workbook['omitted'] = results.omitted
records_to_excel(output_path, workbook)
from outputresult import OutputResult
from outputtype import OutputType
def get_items_to_be_written(repo_path: str, show_commit: bool,
value_regex: Union[str, None] = None) -> ProcessingResult:
value_regex: Union[str, None] = None) -> OutputResult:
"""Determines the contents of '.properties-MERGED' files and writes to a csv file.
Args:
@ -65,28 +42,37 @@ def get_items_to_be_written(repo_path: str, show_commit: bool,
omitted.append(new_entry)
omitted_to_write = [row_header] + omitted if len(omitted) > 0 else None
return ProcessingResult([row_header] + rows, omitted_to_write)
return OutputResult([row_header] + rows, omitted_to_write)
def main():
# noinspection PyTypeChecker
parser = argparse.ArgumentParser(description='Gathers all key-value pairs within .properties-MERGED files into '
'one csv file.',
'one file.',
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
parser.add_argument(dest='output_path', type=str, help='The path to the output csv file. The output path should be'
parser.add_argument(dest='output_path', type=str, help='The path to the output file. The output path should be'
' specified as a relative path with the dot slash notation '
'(i.e. \'./outputpath.csv\') or an absolute path.')
'(i.e. \'./outputpath.xlsx\') or an absolute path.')
parser.add_argument('-r', '--repo', dest='repo_path', type=str, required=False,
help='The path to the repo. If not specified, path of script is used.')
parser.add_argument('-o', '--output-type', dest='output_type', type=OutputType, choices=list(OutputType),
required=False, help="The output type. Currently supports 'csv' or 'xlsx'.", default='xlsx')
parser.add_argument('-nc', '--no_commit', dest='no_commit', action='store_true', default=False,
required=False, help="Suppresses adding commits to the generated csv header.")
required=False, help="Suppresses adding commits to the generated header.")
args = parser.parse_args()
repo_path = args.repo_path if args.repo_path is not None else get_git_root(get_proj_dir())
output_path = args.output_path
show_commit = not args.no_commit
output_type = args.output_type
write_items_to_csv(repo_path, output_path, show_commit)
processing_result = get_items_to_be_written(repo_path, show_commit)
# based on https://stackoverflow.com/questions/60208/replacements-for-switch-statement-in-python
{
OutputType.csv: write_results_to_csv,
OutputType.xlsx: write_results_to_xlsx
}[output_type](processing_result, output_path)
sys.exit(0)

20
release_scripts/localization_scripts/csvutil.py
View File

@ -4,6 +4,9 @@ from typing import List, Iterable, Tuple
import csv
import os
from fileutil import OMITTED_ADDITION, get_filename_addition, DELETED_ADDITION
from outputresult import OutputResult
def records_to_csv(output_path: str, rows: Iterable[List[str]]):
"""Writes rows to a csv file at the specified path.
@ -49,3 +52,20 @@ def csv_to_records(input_path: str, header_row: bool) -> Tuple[List[List[str]],
raise Exception("There was an error parsing csv {path}".format(path=input_path), e)
return results, header
def write_results_to_csv(results: OutputResult, output_path: str):
"""
Writes the result of processing to the output path as a csv file. If omitted values are present, for output_path of
/dir/file.csv, the omitted will be written to /dir/file-omitted.csv.
Args:
results: The results to be written.
output_path: The output path.
"""
records_to_csv(output_path, results.results)
if results.omitted:
records_to_csv(get_filename_addition(output_path, OMITTED_ADDITION), results.omitted)
if results.deleted:
records_to_csv(get_filename_addition(output_path, DELETED_ADDITION), results.deleted)

63
release_scripts/localization_scripts/diffscript.py
View File

@ -2,47 +2,22 @@
and generates a csv file containing the items changed. This script requires the python libraries:
gitpython and jproperties. As a consequence, it also requires git >= 1.7.0 and python >= 3.4.
"""
import collections
import re
import sys
from envutil import get_proj_dir
from excelutil import records_to_excel
from fileutil import get_filename_addition, OMITTED_ADDITION, DELETED_ADDITION
from excelutil import write_results_to_xlsx
from gitutil import get_property_files_diff, get_commit_id, get_git_root
from itemchange import ItemChange, ChangeType
from csvutil import records_to_csv
from csvutil import write_results_to_csv
import argparse
from typing import Union, TypedDict, List
from typing import Union
from langpropsutil import get_commit_for_language, LANG_FILENAME
class DiffResult:
results: List[List[str]]
deleted: Union[List[List[str]], None]
omitted: Union[List[List[str]], None]
def write_items_to_csv(results: DiffResult, output_path: str):
records_to_csv(output_path, results.results)
if results.deleted:
records_to_csv(get_filename_addition(output_path, DELETED_ADDITION), results.deleted)
if results.omitted:
records_to_csv(get_filename_addition(output_path, OMITTED_ADDITION), results.omitted)
def write_items_to_xlsx(results: DiffResult, output_path: str):
workbook = collections.OrderedDict([('results', results.results)])
if results.deleted:
workbook['deleted'] = results.deleted
if results.omitted:
workbook['omitted'] = results.omitted
records_to_excel(output_path, workbook)
from outputresult import OutputResult
from outputtype import OutputType
def get_diff_to_write(repo_path: str, commit_1_id: str, commit_2_id: str, show_commits: bool, separate_deleted: bool,
value_regex: Union[str, None] = None) -> DiffResult:
value_regex: Union[str, None] = None) -> OutputResult:
"""Determines the changes made in '.properties-MERGED' files from one commit to another commit and returns results.
Args:
@ -66,7 +41,7 @@ def get_diff_to_write(repo_path: str, commit_1_id: str, commit_2_id: str, show_c
for entry in get_property_files_diff(repo_path, commit_1_id, commit_2_id):
entry_row = entry.get_row()
if entry.type == ChangeType.DELETION:
if separate_deleted and entry.type == ChangeType.DELETION:
deleted.append(entry_row)
if value_regex is not None and re.match(value_regex, entry.cur_val):
omitted.append(entry_row)
@ -76,13 +51,7 @@ def get_diff_to_write(repo_path: str, commit_1_id: str, commit_2_id: str, show_c
omitted_result = [row_header] + omitted if len(omitted) > 0 else None
deleted_result = [row_header] + deleted if len(deleted) > 0 else None
return {
'results': [row_header] + rows,
'omitted': omitted_result,
'deleted': deleted_result
}
return OutputResult([row_header] + rows, omitted_result, deleted_result)
def main():
@ -91,9 +60,9 @@ def main():
"'.properties-MERGED' files and generates a csv file containing "
"the items changed.",
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
parser.add_argument(dest='output_path', type=str, help='The path to the output csv file. The output path should '
parser.add_argument(dest='output_path', type=str, help='The path to the output file. The output path should '
'be specified as a relative path with the dot slash notation'
' (i.e. \'./outputpath.csv\') or an absolute path.')
' (i.e. \'./outputpath.xlsx\') or an absolute path.')
parser.add_argument('-r', '--repo', dest='repo_path', type=str, required=False,
help='The path to the repo. If not specified, path of script is used.')
@ -104,6 +73,8 @@ def main():
help='The commit for current release.')
parser.add_argument('-nc', '--no-commits', dest='no_commits', action='store_true', default=False,
required=False, help="Suppresses adding commits to the generated csv header.")
parser.add_argument('-o', '--output-type', dest='output_type', type=OutputType, choices=list(OutputType),
required=False, help="The output type. Currently supports 'csv' or 'xlsx'.", default='xlsx')
parser.add_argument('-l', '--language', dest='language', type=str, default=None, required=False,
help='Specify the language in order to determine the first commit to use (i.e. \'ja\' for '
'Japanese. This flag overrides the first-commit flag.')
@ -112,6 +83,8 @@ def main():
repo_path = args.repo_path if args.repo_path is not None else get_git_root(get_proj_dir())
output_path = args.output_path
commit_1_id = args.commit_1_id
output_type = args.output_type
lang = args.language
if lang is not None:
commit_1_id = get_commit_for_language(lang)
@ -125,7 +98,13 @@ def main():
commit_2_id = args.commit_2_id
show_commits = not args.no_commits
write_diff_to_csv(repo_path, output_path, commit_1_id, commit_2_id, show_commits)
processing_result = get_diff_to_write(repo_path, commit_1_id, commit_2_id, show_commits)
# based on https://stackoverflow.com/questions/60208/replacements-for-switch-statement-in-python
{
OutputType.csv: write_results_to_csv,
OutputType.xlsx: write_results_to_xlsx
}[output_type](processing_result, output_path)
sys.exit(0)

41
release_scripts/localization_scripts/excelutil.py
View File

@ -1,10 +1,22 @@
"""Provides tools for parsing and writing to a csv file.
"""
from typing import List, Dict, OrderedDict
import collections
from typing import List, OrderedDict
import pyexcel
from outputresult import OutputResult
Workbook = OrderedDict[str, List[List[str]]]
# The name for the results sheet
RESULTS_SHEET_NAME = 'results'
# The name for the sheet of deleted items
DELETED_SHEET_NAME = 'deleted'
# The name for the sheet of omitted items
OMITTED_SHEET_NAME = 'omitted'
def records_to_excel(output_path: str, workbook: Workbook):
"""Writes workbook data model to an excel workbook.
@ -22,13 +34,32 @@ def records_to_excel(output_path: str, workbook: Workbook):
)
def csv_to_records(input_path: str) -> Workbook:
"""Reads rows to a csv file at the specified path.
def excel_to_records(input_path: str) -> Workbook:
"""Reads rows to a excel file at the specified path.
Args:
input_path (str): The path where the csv file will be written.
input_path (str): The path where the excel file will be read.
"""
return pyexcel.get_book_dict(
file_name=input_path
)
)
def write_results_to_xlsx(results: OutputResult, output_path: str):
"""
Writes the result of processing to the output path as a xlsx file. Results will be written to a 'results' sheet.
Omitted results will be written to an 'omitted' sheet. Deleted results will be written to a 'deleted' sheet.
Args:
results: The results to be written.
output_path: The output path.
"""
workbook = collections.OrderedDict([(RESULTS_SHEET_NAME, results.results)])
if results.omitted:
workbook[OMITTED_SHEET_NAME] = results.omitted
if results.deleted:
workbook[DELETED_SHEET_NAME] = results.deleted
records_to_excel(output_path, workbook)

3
release_scripts/localization_scripts/fileutil.py
View File

@ -44,7 +44,8 @@ def get_new_path(orig_path: str, new_filename: str) -> str:
OMITTED_ADDITION = '-omitted'
# For use with creating csv filenames for entries that have been deleted.
OMITTED_ADDITION = '-deleted'
DELETED_ADDITION = '-deleted'
def get_filename_addition(orig_path: str, filename_addition: str) -> str:
"""Gets filename with addition. So if item is '/path/name.ext' and the filename_addition is '-add', the new result

28
release_scripts/localization_scripts/outputresult.py Normal file
View File

@ -0,0 +1,28 @@
from typing import List, Union
class OutputResult:
"""
Describes a result that is ready to be written to file(s).
"""
results: List[List[str]]
omitted: Union[List[List[str]], None]
deleted: Union[List[List[str]], None]
def __init__(self, results: List[List[str]], omitted: Union[List[List[str]], None] = None,
deleted: Union[List[List[str]], None] = None):
"""
Constructs a ProcessingResult.
Args:
results: Items to be written as results. Data will be written such that the item at row,cell will be
located within result at results[row][col].
omitted: Items to be written as omitted. Data will be written such that the item at row,cell will be
located within result at results[row][col].
deleted: Items to be written as omitted. Data will be written such that the item at row,cell will be
located within result at results[row][col].
"""
self.results = results
self.omitted = omitted
self.deleted = deleted

10
release_scripts/localization_scripts/outputtype.py Normal file
View File

@ -0,0 +1,10 @@
from enum import Enum
class OutputType(Enum):
"""Describes the output file type."""
xlsx = 'xlsx'
csv = 'csv'
def __str__(self):
return str(self.value)