mstoeck3/autopsy-flatpak
1
0
Fork 0
mirror of https://github.com/overcuriousity/autopsy-flatpak.git synced 2025-07-06 21:00:22 +00:00
Code Issues Packages Projects Releases Wiki Activity

module parsing

Browse Source
This commit is contained in:
Greg DiCristofaro 2023-08-29 12:11:15 -04:00
parent 84814d7f8b
commit 9906d5dc47
8 changed files with 537 additions and 196 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
release_scripts/APIUpdate/nbactions.xml
View File

@ -52,4 +52,4 @@
<exec.appArgs>--help</exec.appArgs>
</properties>
</action>
</actions>
</actions>

34
release_scripts/APIUpdate/pom.xml
View File

@ -24,5 +24,39 @@
<version>1.5.0</version>
</dependency>
<!-- https://mvnrepository.com/artifact/org.apache.commons/commons-lang3 -->
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-lang3</artifactId>
<version>3.13.0</version>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<artifactId>maven-assembly-plugin</artifactId>
<configuration>
<archive>
<manifest>
<mainClass>org.sleuthkit.autopsy.apiupdate.Main</mainClass>
</manifest>
</archive>
<descriptorRefs>
<descriptorRef>jar-with-dependencies</descriptorRef>
</descriptorRefs>
</configuration>
<executions>
<execution>
<id>make-assembly</id> <!-- this is used for inheritance merges -->
<phase>package</phase> <!-- bind to the packaging phase -->
<goals>
<goal>single</goal>
</goals>
</execution>
</executions>
</plugin>
</plugins>
</build>
</project>

70
release_scripts/APIUpdate/src/main/java/org/sleuthkit/autopsy/apiupdate/APIDiff.java Normal file
View File

@ -0,0 +1,70 @@
/*
* Click nbfs://nbhost/SystemFileSystem/Templates/Licenses/license-default.txt to change this license
* Click nbfs://nbhost/SystemFileSystem/Templates/Classes/Class.java to edit this template
*/
package org.sleuthkit.autopsy.apiupdate;
import japicmp.cmp.JApiCmpArchive;
import japicmp.cmp.JarArchiveComparator;
import japicmp.cmp.JarArchiveComparatorOptions;
import japicmp.model.JApiClass;
import java.io.File;
import java.io.FileFilter;
import java.io.IOException;
import java.text.MessageFormat;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import java.util.stream.Collectors;
import java.util.stream.Stream;
/**
*
* @author gregd
*/
public class APIDiff {
private static final FileFilter JAR_FILTER
= (File f) -> f.isFile() && (f.getName().toLowerCase().endsWith(".jar") || f.getName().toLowerCase().endsWith(".nbm"));
private static List<String> getCommonJars(File prevDir, File currDir) {
Set<String> prevJars = getJars(prevDir);
Set<String> currJars = getJars(currDir);
Set<String> commonJars = new HashSet<>(prevJars);
commonJars.retainAll(currJars);
// TODO how to handle different
return commonJars.stream().sorted().collect(Collectors.toList());
}
private static Set<String> getJars(File dir) {
return Stream.of(dir.listFiles(JAR_FILTER))
.map(f -> f.getName())
.collect(Collectors.toSet());
}
private static Set<String> getPublicPackages(File jarFile) throws IOException, IllegalStateException {
String publicPackageStr = ManifestLoader.loadFromJar(jarFile).getValue("OpenIDE-Module-Public-Packages");
if (publicPackageStr == null) {
throw new IllegalStateException(MessageFormat.format("Manifest for {0} does not have key of 'OpenIDE-Module-Public-Packages'", jarFile.getAbsolutePath()));
} else {
return Stream.of(publicPackageStr.split(","))
.map(String::trim)
.map(str -> str.endsWith(".*") ? str.substring(0, str.length() - 2) : str)
.collect(Collectors.toSet());
}
}
private static void getComparison(String prevVersion, String curVersion, File prevJar, File curJar) {
JarArchiveComparatorOptions comparatorOptions = new JarArchiveComparatorOptions();
comparatorOptions.getIgnoreMissingClasses().setIgnoreAllMissingClasses(true);
JarArchiveComparator jarArchiveComparator = new JarArchiveComparator(comparatorOptions);
List<JApiClass> jApiClasses = jarArchiveComparator.compare(
new JApiCmpArchive(prevJar, prevVersion),
new JApiCmpArchive(curJar, curVersion)
);
System.out.println("Comparing " + prevJar.getName());
System.out.println(jApiClasses);
}
}

28
release_scripts/APIUpdate/src/main/java/org/sleuthkit/autopsy/classpathsimplication/apiupdate/CLIProcessor.java → release_scripts/APIUpdate/src/main/java/org/sleuthkit/autopsy/apiupdate/CLIProcessor.java
View File

@ -2,7 +2,7 @@
* Click nbfs://nbhost/SystemFileSystem/Templates/Licenses/license-default.txt to change this license
* Click nbfs://nbhost/SystemFileSystem/Templates/Classes/Class.java to edit this template
*/
package org.sleuthkit.autopsy.classpathsimplication.apiupdate;
package org.sleuthkit.autopsy.apiupdate;
import java.io.File;
import java.util.Arrays;
@ -58,11 +58,21 @@ public class CLIProcessor {
.required(true)
.build();
static Option SRC_LOC_OPT = Option.builder()
.argName("path")
.desc("The path to the root of the autopsy repor")
.hasArg(true)
.longOpt("src-path")
.option("s")
.required(true)
.build();
static List<Option> ALL_OPTIONS = Arrays.asList(
PREV_VERS_PATH_OPT,
CUR_VERS_PATH_OPT,
PREV_VERS_OPT,
CUR_VERS_OPT
CUR_VERS_OPT,
SRC_LOC_OPT
);
static Options CLI_OPTIONS = getCliOptions(ALL_OPTIONS);
@ -103,7 +113,7 @@ public class CLIProcessor {
CommandLine helpCmd = parser.parse(HELP_OPTIONS, args, true);
boolean isHelp = helpCmd.hasOption(HELP_OPT);
if (isHelp) {
return new CLIArgs(null, null, null, null, true);
return new CLIArgs(null, null, null, null, null, true);
}
CommandLine cmd = parser.parse(CLI_OPTIONS, args);
@ -111,8 +121,10 @@ public class CLIProcessor {
String prevVers = cmd.getOptionValue(PREV_VERS_OPT);
String curVersPath = cmd.getOptionValue(CUR_VERS_PATH_OPT);
String prevVersPath = cmd.getOptionValue(PREV_VERS_PATH_OPT);
String srcPath = cmd.getOptionValue(SRC_LOC_OPT);
File curVersFile = new File(curVersPath);
File prevVersFile = new File(prevVersPath);
File srcPathFile = new File(srcPath);
if (!curVersFile.isDirectory()) {
throw new ParseException("No directory found at " + curVersFile.getAbsolutePath());
@ -122,7 +134,11 @@ public class CLIProcessor {
throw new ParseException("No directory found at " + prevVersFile.getAbsolutePath());
}
return new CLIArgs(curVers, prevVers, curVersFile, prevVersFile, false);
if (!srcPathFile.isDirectory()) {
throw new ParseException("No directory found at " + srcPathFile.getAbsolutePath());
}
return new CLIArgs(curVers, prevVers, curVersFile, prevVersFile, srcPathFile, false);
}
@ -133,6 +149,7 @@ public class CLIProcessor {
private final File currentVersPath;
private final File previousVersPath;
private final boolean isHelp;
private final File srcPath;
public String getCurrentVersion() {
return currentVersion;
@ -154,11 +171,12 @@ public class CLIProcessor {
return isHelp;
}
public CLIArgs(String currentVersion, String previousVersion, File currentVersPath, File previousVersPath, boolean isHelp) {
public CLIArgs(String currentVersion, String previousVersion, File currentVersPath, File previousVersPath, File srcPath, boolean isHelp) {
this.currentVersion = currentVersion;
this.previousVersion = previousVersion;
this.currentVersPath = currentVersPath;
this.previousVersPath = previousVersPath;
this.srcPath = srcPath;
this.isHelp = isHelp;
}

82
release_scripts/APIUpdate/src/main/java/org/sleuthkit/autopsy/apiupdate/Main.java Normal file
View File

@ -0,0 +1,82 @@
/*
* Click nbfs://nbhost/SystemFileSystem/Templates/Licenses/license-default.txt to change this license
*/
package org.sleuthkit.autopsy.apiupdate;
import japicmp.cmp.JApiCmpArchive;
import japicmp.cmp.JarArchiveComparator;
import japicmp.cmp.JarArchiveComparatorOptions;
import japicmp.model.JApiClass;
import java.io.File;
import java.io.FileFilter;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStream;
import java.text.MessageFormat;
import java.util.Enumeration;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import java.util.jar.Attributes;
import java.util.jar.Manifest;
import java.util.logging.Level;
import java.util.logging.Logger;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import java.util.zip.ZipEntry;
import java.util.zip.ZipFile;
import org.apache.commons.cli.ParseException;
import org.sleuthkit.autopsy.apiupdate.CLIProcessor.CLIArgs;
/**
*
* @author gregd
*/
public class Main {
public static void main(String[] args) {
args = "-c C:\\Users\\gregd\\Documents\\Source\\autopsy\\build\\cluster\\modules -p C:\\Users\\gregd\\Desktop\\prevVers -cv 4.21.0 -pv 4.20.0".split(" ");
CLIArgs cliArgs;
try {
cliArgs = CLIProcessor.parseCli(args);
if (cliArgs.isIsHelp()) {
CLIProcessor.printHelp(null);
System.exit(0);
}
} catch (ParseException ex) {
CLIProcessor.printHelp(ex);
System.exit(-1);
return;
}
// for (String commonJarFileName : getCommonJars(cliArgs.getPreviousVersPath(), cliArgs.getCurrentVersPath())) {
//// getComparison(
//// cliArgs.getPreviousVersion(),
//// cliArgs.getCurrentVersion(),
//// cliArgs.getPreviousVersPath().toPath().resolve(commonJarFileName).toFile(),
//// cliArgs.getCurrentVersPath().toPath().resolve(commonJarFileName).toFile());
// try {
// Set<String> pubPackages = getPublicPackages(cliArgs.getPreviousVersPath().toPath().resolve(commonJarFileName).toFile());
// System.out.println(pubPackages);
// } catch (IOException ex) {
// Logger.getLogger(Main.class.getName()).log(Level.SEVERE, null, ex);
// } catch (IllegalStateException ex) {
// Logger.getLogger(Main.class.getName()).log(Level.SEVERE, null, ex);
// }
// }
}
private static void mainRun() {
// get public API diff's, for each jar
// limit to public packages
// one of the following:
// generate text output of difference
// update version numbers in manifest file/references accordingly
}
}

42
release_scripts/APIUpdate/src/main/java/org/sleuthkit/autopsy/apiupdate/ManifestLoader.java Normal file
View File

@ -0,0 +1,42 @@
/*
* Click nbfs://nbhost/SystemFileSystem/Templates/Licenses/license-default.txt to change this license
* Click nbfs://nbhost/SystemFileSystem/Templates/Classes/Class.java to edit this template
*/
package org.sleuthkit.autopsy.apiupdate;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStream;
import java.util.Enumeration;
import java.util.jar.Attributes;
import java.util.jar.Manifest;
import java.util.zip.ZipEntry;
import java.util.zip.ZipFile;
/**
*
* @author gregd
*/
public class ManifestLoader {
public static Attributes loadInputStream(InputStream is) throws IOException {
Manifest manifest = new Manifest(is);
return manifest.getMainAttributes();
}
public static Attributes loadFromJar(File jarFile) throws IOException {
ZipFile zipFile = new ZipFile(jarFile);
Enumeration<? extends ZipEntry> entries = zipFile.entries();
while (entries.hasMoreElements()) {
ZipEntry entry = entries.nextElement();
if ("META-INF/MANIFEST.MF".equalsIgnoreCase(entry.getName())) {
return loadInputStream(zipFile.getInputStream(entry));
}
}
throw new FileNotFoundException("Could not find MANIFEST.MF in " + jarFile.getAbsolutePath());
}
}

232
release_scripts/APIUpdate/src/main/java/org/sleuthkit/autopsy/apiupdate/ModuleUpdates.java Normal file
View File

@ -0,0 +1,232 @@
/*
* Click nbfs://nbhost/SystemFileSystem/Templates/Licenses/license-default.txt to change this license
* Click nbfs://nbhost/SystemFileSystem/Templates/Classes/Class.java to edit this template
*/
package org.sleuthkit.autopsy.apiupdate;
import java.io.File;
import java.io.IOException;
import java.text.MessageFormat;
import java.text.ParseException;
import java.util.jar.Attributes;
import java.util.logging.Level;
import java.util.logging.Logger;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import org.apache.commons.lang3.StringUtils;
/**
*
* @author gregd
*/
public class ModuleUpdates {
private static final Logger LOGGER = Logger.getLogger(ModuleUpdates.class.getName());
// // Spec
// // project.properties
// // spec.version.base
// // manifest
// // OpenIDE-Module-Specification-Version
// // Implementation
// // manifest
// // OpenIDE-Module-Implementation-Version
// // Release
// // manifest
// // OpenIDE-Module (/number)
//
// // Dependency specification
// // project.xml
// // project.configuration.data.module-dependencies.dependency.run-dependency:
// // specification-version
// // release-version
private static final Pattern SPEC_REGEX = Pattern.compile("^\\s*(?<major>\\d*)\\.(?<minor>\\d*)(\\.(?<patch>\\d*))?\\s*$");
private static final String SPEC_KEY = "OpenIDE-Module-Specification-Version";
private static final String IMPL_KEY = "OpenIDE-Module-Implementation-Version";
private static final String RELEASE_KEY = "OpenIDE-Module";
private static final Pattern RELEASE_REGEX = Pattern.compile("^\\s*(?<releaseName>.+?)(/(?<releaseNum>\\d*))?\\s*$");
private static final SemVer DEFAULT_SEMVER = new SemVer(1, 0, null);
private static final int DEFAULT_VERS_VAL = 1;
private static SemVer parseSemVer(String semVerStr, SemVer defaultSemVer, String resourceForLogging) {
if (StringUtils.isBlank(semVerStr)) {
return defaultSemVer;
}
Matcher m = SPEC_REGEX.matcher(semVerStr);
if (m.find()) {
try {
int major = Integer.parseInt(m.group("major"));
int minor = Integer.parseInt(m.group("minor"));
String patchStr = m.group("patch");
Integer patch = StringUtils.isBlank(patchStr) ? null : Integer.parseInt(patchStr);
return new SemVer(major, minor, patch);
} catch (NullPointerException | NumberFormatException ex) {
LOGGER.log(Level.SEVERE, MessageFormat.format("Unable to parse semver string {0} for {1}", semVerStr, resourceForLogging), ex);
return defaultSemVer;
}
} else {
return defaultSemVer;
}
}
private static ReleaseVal parseReleaseVers(String releaseStr, ReleaseVal defaultVal, String resourceForLogging) {
if (StringUtils.isBlank(releaseStr)) {
return defaultVal;
}
Matcher m = RELEASE_REGEX.matcher(releaseStr);
if (m.find()) {
try {
int major = Integer.parseInt(m.group("major"));
int minor = Integer.parseInt(m.group("minor"));
String patchStr = m.group("patch");
Integer patch = StringUtils.isBlank(patchStr) ? null : Integer.parseInt(patchStr);
return new SemVer(major, minor, patch);
} catch (NullPointerException | NumberFormatException ex) {
LOGGER.log(Level.SEVERE, MessageFormat.format("Unable to parse semver string {0} for {1}", releaseStr, resourceForLogging), ex);
return defaultSemVer;
}
} else {
return defaultSemVer;
}
}
private static int tryParse(String str, int defaultVal, String resourceForLogging) {
try {
return Integer.parseInt(str);
} catch (NullPointerException | NumberFormatException ex) {
LOGGER.log(Level.SEVERE, MessageFormat.format("Unable to parse version string {0} for {1}", str, resourceForLogging), ex);
return defaultVal;
}
}
public static SemVer getPrevVersions(File jarFile) throws IOException {
Attributes manifest = ManifestLoader.loadFromJar(jarFile);
String spec = manifest.getValue(SPEC_KEY);
SemVer specSemVer = parseSemVer(spec, DEFAULT_SEMVER,
MessageFormat.format("{0} in manifest for {1}", SPEC_KEY, jarFile));
int implementation = tryParse(manifest.getValue(IMPL_KEY), DEFAULT_VERS_VAL,
MessageFormat.format("{0} in manifest for {1}", IMPL_KEY, jarFile));
// // Spec
// // project.properties
// // spec.version.base
// // manifest
// // OpenIDE-Module-Specification-Version
// // Implementation
// // manifest
// // OpenIDE-Module-Implementation-Version
// // Release
// // manifest
// // OpenIDE-Module (/number)
//
// // Dependency specification
// // project.xml
// // project.configuration.data.module-dependencies.dependency.run-dependency:
// // specification-version
// // release-version
}
private static void updateVersions() {
// [specification major/minor/patch, implementation, release]
// assumed defaults???
// NON_COMPATIBLE:
// specification.major += 1
// implementation += 1
// release += 1
// COMPATIBLE:
// specification.minor += 1
// implementation += 1
// NO_CHANGES:
// implementation += 1
}
public static class ReleaseVal {
private final String moduleName;
private final Integer releaseVersion;
public ReleaseVal(String moduleName, Integer releaseVersion) {
this.moduleName = moduleName;
this.releaseVersion = releaseVersion;
}
public String getModuleName() {
return moduleName;
}
public Integer getReleaseVersion() {
return releaseVersion;
}
public String getFullReleaseStr() {
return this.releaseVersion == null
? moduleName
: MessageFormat.format("{0}/{1,number,#}", moduleName, releaseVersion);
}
}
public static class SemVer {
private final int major;
private final int minor;
private final Integer patch;
public SemVer(int major, int minor, Integer patch) {
this.major = major;
this.minor = minor;
this.patch = patch;
}
public int getMajor() {
return major;
}
public int getMinor() {
return minor;
}
public Integer getPatch() {
return patch;
}
public String getSemVerStr() {
return (patch == null)
? MessageFormat.format("{0,number,#}.{1,number,#}", major, minor)
: MessageFormat.format("{0,number,#}.{1,number,#}.{2,number,#}", major, minor, patch);
}
}
public static class ModuleVersionNumbers {
private final SemVer spec;
private final int implementation;
private final int release;
public ModuleVersionNumbers(SemVer spec, int implementation, int release) {
this.spec = spec;
this.implementation = implementation;
this.release = release;
}
public SemVer getSpec() {
return spec;
}
public int getImplementation() {
return implementation;
}
public int getRelease() {
return release;
}
}
}

137
release_scripts/APIUpdate/src/main/java/org/sleuthkit/autopsy/classpathsimplication/apiupdate/APIUpdate.java
View File

@ -1,137 +0,0 @@
/*
* Click nbfs://nbhost/SystemFileSystem/Templates/Licenses/license-default.txt to change this license
*/
package org.sleuthkit.autopsy.classpathsimplication.apiupdate;
import japicmp.cmp.JApiCmpArchive;
import japicmp.cmp.JarArchiveComparator;
import japicmp.cmp.JarArchiveComparatorOptions;
import japicmp.model.JApiClass;
import java.io.File;
import java.io.FileFilter;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStream;
import java.text.MessageFormat;
import java.util.Enumeration;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import java.util.jar.Attributes;
import java.util.jar.Manifest;
import java.util.logging.Level;
import java.util.logging.Logger;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import java.util.zip.ZipEntry;
import java.util.zip.ZipFile;
import org.apache.commons.cli.ParseException;
import org.sleuthkit.autopsy.classpathsimplication.apiupdate.CLIProcessor.CLIArgs;
/**
*
* @author gregd
*/
public class APIUpdate {
public static void main(String[] args) {
args = "-c C:\\Users\\gregd\\Documents\\Source\\autopsy\\build\\cluster\\modules -p C:\\Users\\gregd\\Desktop\\prevVers -cv 4.21.0 -pv 4.20.0".split(" ");
CLIArgs cliArgs;
try {
cliArgs = CLIProcessor.parseCli(args);
if (cliArgs.isIsHelp()) {
CLIProcessor.printHelp(null);
System.exit(0);
}
} catch (ParseException ex) {
CLIProcessor.printHelp(ex);
System.exit(-1);
return;
}
for (String commonJarFileName : getCommonJars(cliArgs.getPreviousVersPath(), cliArgs.getCurrentVersPath())) {
// getComparison(
// cliArgs.getPreviousVersion(),
// cliArgs.getCurrentVersion(),
// cliArgs.getPreviousVersPath().toPath().resolve(commonJarFileName).toFile(),
// cliArgs.getCurrentVersPath().toPath().resolve(commonJarFileName).toFile());
try {
Set<String> pubPackages = getPublicPackages(cliArgs.getPreviousVersPath().toPath().resolve(commonJarFileName).toFile());
System.out.println(pubPackages);
} catch (IOException ex) {
Logger.getLogger(APIUpdate.class.getName()).log(Level.SEVERE, null, ex);
} catch (IllegalStateException ex) {
Logger.getLogger(APIUpdate.class.getName()).log(Level.SEVERE, null, ex);
}
}
}
private static final FileFilter JAR_FILTER
= (File f) -> f.isFile() && (f.getName().toLowerCase().endsWith(".jar") || f.getName().toLowerCase().endsWith(".nbm"));
private static List<String> getCommonJars(File prevDir, File currDir) {
Set<String> prevJars = getJars(prevDir);
Set<String> currJars = getJars(currDir);
Set<String> commonJars = new HashSet<>(prevJars);
commonJars.retainAll(currJars);
// TODO how to handle different
return commonJars.stream().sorted().collect(Collectors.toList());
}
private static Set<String> getJars(File dir) {
return Stream.of(dir.listFiles(JAR_FILTER))
.map(f -> f.getName())
.collect(Collectors.toSet());
}
private static Set<String> getPublicPackages(File jarFile) throws IOException, IllegalStateException {
ZipFile zipFile = new ZipFile(jarFile);
Enumeration<? extends ZipEntry> entries = zipFile.entries();
while (entries.hasMoreElements()) {
ZipEntry entry = entries.nextElement();
if ("META-INF/MANIFEST.MF".equalsIgnoreCase(entry.getName())) {
InputStream stream = zipFile.getInputStream(entry);
Manifest manifest = new Manifest(stream);
Attributes attributes = manifest.getMainAttributes();
String publicPackageStr = attributes.getValue("OpenIDE-Module-Public-Packages");
if (publicPackageStr == null) {
throw new IllegalStateException(MessageFormat.format("Manifest for {0} does not have key of 'OpenIDE-Module-Public-Packages'", jarFile.getAbsolutePath()));
} else {
return Stream.of(publicPackageStr.split(","))
.map(String::trim)
.map(str -> str.endsWith(".*") ? str.substring(0, str.length() - 2) : str)
.collect(Collectors.toSet());
}
}
}
throw new FileNotFoundException("Could not find MANIFEST.MF in " + jarFile.getAbsolutePath());
}
private static void getComparison(String prevVersion, String curVersion, File prevJar, File curJar) {
JarArchiveComparatorOptions comparatorOptions = new JarArchiveComparatorOptions();
comparatorOptions.getIgnoreMissingClasses().setIgnoreAllMissingClasses(true);
JarArchiveComparator jarArchiveComparator = new JarArchiveComparator(comparatorOptions);
List<JApiClass> jApiClasses = jarArchiveComparator.compare(
new JApiCmpArchive(prevJar, prevVersion),
new JApiCmpArchive(curJar, curVersion)
);
System.out.println("Comparing " + prevJar.getName());
System.out.println(jApiClasses);
}
private static void mainRun() {
// get public API diff's, for each jar
// limit to public packages
// one of the following:
// generate text output of difference
// update version numbers in manifest file/references accordingly
}
}