Add optional on-device file hashing

README.md

@@ -118,7 +118,7 @@ The following data can be extracted:
 | A list of all packages installed and related distribution files. | |  `packages.json` |
 | Copy of all installed APKs or of only those not marked as system apps. | ✅ | `apks/*` |
 | Intrusion Logging logs. Contains private data such as navigation history. | ✅ | `intrusion_logs/*` |
-| A list of files on the system. | | `files.json` |
+| A list of files on the system, optionally including on-device hashes. | :white_check_mark: | `files.json` |
 | A copy of the files available in temp folders. | | `tmp/*` |
 | A bug report containing system and app-specific logs, with no private data included. | | `bugreport.zip` |
 
@@ -200,6 +200,21 @@ Would you like to take the Intrusion Logs of the device?
 | Yes | Intrusion Logs will be retrieved from the phone. |
 | No | Intrusion Logs acquisition is skipped. |
 
+### Hashing files on the device
+
+```
+Would you like to hash files on the device? This is resource-intensive and may cause the collector to stop on some devices.
+
+? Hash files:
+  ▸ No
+    Yes
+```
+
+Selecting `Yes` adds MD5, SHA-1, SHA-256 and SHA-512 hashes to the entries in
+`files.json` where hashing is supported. This performs the hashing on the
+device and can take a long time or cause the collector to stop on devices with
+limited resources. The default `No` option only collects file metadata.
+
 ### Unattended acquisitions
 
 Every prompt can be answered ahead of time with a command-line flag. A flag that is not passed keeps prompting interactively as before.
@@ -210,11 +225,12 @@ Every prompt can be answered ahead of time with a command-line flag. A flag that
 | `-download` / `-d` | `all`, `non-system`, `none` | [Downloading copies of apps](#downloading-copies-of-apps) |
 | `-remove-trusted` / `-r` | `yes`, `no` | [Removing apps signed with a trusted certificate](#removing-apps-signed-with-a-trusted-certificate), ignored with `-download none` |
 | `-intrusion-logs` / `-i` | `yes`, `no` | [Intrusion Logs](#intrusion-logs) |
+| `-hash-files` / `-H` | `yes`, `no` | [Hashing files on the device](#hashing-files-on-the-device) |
 
 With `-non-interactive` (`-n`), androidqf never prompts: it fails before the acquisition starts if one of the flags above is missing, fails if multiple devices are attached and no `-serial` is given, and skips the final "Press Enter to finish". A fully unattended run looks like this:
 
 ```bash
-androidqf -serial <serial> -backup none -download all -remove-trusted no -intrusion-logs no -non-interactive
+androidqf -serial <serial> -backup none -download all -remove-trusted no -intrusion-logs no -hash-files no -non-interactive
 ```
 
 > [!NOTE]

main.go

@@ -122,7 +122,7 @@ func errorOnDeviceSelection([]deviceMenuItem) (string, error) {
 	return "", fmt.Errorf("multiple devices detected, use -serial to select one")
 }
 
-func buildOptions(fast, nonInteractive bool, backup, download, removeTrusted, intrusionLogs, moduleFilter string) (*modules.Options, error) {
+func buildOptions(fast, nonInteractive bool, backup, download, removeTrusted, intrusionLogs, hashFiles, moduleFilter string) (*modules.Options, error) {
 	opts := &modules.Options{Fast: fast, NonInteractive: nonInteractive}
 	var err error
 	if backup != "" {
@@ -145,6 +145,11 @@ func buildOptions(fast, nonInteractive bool, backup, download, removeTrusted, in
 			return nil, err
 		}
 	}
+	if hashFiles != "" {
+		if opts.HashFiles, err = modules.ParseHashFilesOption(hashFiles); err != nil {
+			return nil, err
+		}
+	}
 	if err = modules.ValidateNonInteractive(opts, moduleFilter); err != nil {
 		return nil, err
 	}
@@ -165,6 +170,7 @@ func main() {
 	var downloadFlag string
 	var removeTrustedFlag string
 	var intrusionLogsFlag string
+	var hashFilesFlag string
 	var nonInteractive bool
 
 	// Command line options
@@ -190,6 +196,8 @@ func main() {
 	flag.StringVar(&removeTrustedFlag, "r", "", "Answer the trusted-APK removal prompt: yes or no (ignored with -download none)")
 	flag.StringVar(&intrusionLogsFlag, "intrusion-logs", "", "Answer the Intrusion Logs prompt: yes or no (yes still requires taps on the device to download new logs)")
 	flag.StringVar(&intrusionLogsFlag, "i", "", "Answer the Intrusion Logs prompt: yes or no (yes still requires taps on the device to download new logs)")
+	flag.StringVar(&hashFilesFlag, "hash-files", "", "Answer the on-device file hashing prompt: yes or no (resource-intensive)")
+	flag.StringVar(&hashFilesFlag, "H", "", "Answer the on-device file hashing prompt: yes or no (resource-intensive)")
 	flag.BoolVar(&nonInteractive, "non-interactive", false, "Never prompt: fail if a prompt would be reached without its flag and skip the final 'Press Enter'")
 	flag.BoolVar(&nonInteractive, "n", false, "Never prompt: fail if a prompt would be reached without its flag and skip the final 'Press Enter'")
 	flag.BoolVar(&version_flag, "version", false, "Show version")
@@ -213,7 +221,7 @@ func main() {
 		os.Exit(0)
 	}
 
-	opts, err := buildOptions(fast, nonInteractive, backupFlag, downloadFlag, removeTrustedFlag, intrusionLogsFlag, module)
+	opts, err := buildOptions(fast, nonInteractive, backupFlag, downloadFlag, removeTrustedFlag, intrusionLogsFlag, hashFilesFlag, module)
 	if err != nil {
 		log.Fatal(err)
 	}

main_options_test.go

@@ -8,7 +8,7 @@ import (
 )
 
 func TestBuildOptionsNoFlagsKeepsInteractiveDefaults(t *testing.T) {
-	opts, err := buildOptions(false, false, "", "", "", "", "")
+	opts, err := buildOptions(false, false, "", "", "", "", "", "")
 	if err != nil {
 		t.Fatalf("unexpected error: %v", err)
 	}
@@ -19,18 +19,19 @@ func TestBuildOptionsNoFlagsKeepsInteractiveDefaults(t *testing.T) {
 
 func TestBuildOptionsInvalidValueFails(t *testing.T) {
 	tests := []struct {
-		name                                           string
-		backup, download, removeTrusted, intrusionLogs string
-		wantErr                                        string
+		name                                                      string
+		backup, download, removeTrusted, intrusionLogs, hashFiles string
+		wantErr                                                   string
 	}{
-		{"backup", "maybe", "", "", "", "invalid -backup value"},
-		{"download", "", "some", "", "", "invalid -download value"},
-		{"remove-trusted", "", "", "nope", "", "invalid -remove-trusted value"},
-		{"intrusion-logs", "", "", "", "never", "invalid -intrusion-logs value"},
+		{"backup", "maybe", "", "", "", "", "invalid -backup value"},
+		{"download", "", "some", "", "", "", "invalid -download value"},
+		{"remove-trusted", "", "", "nope", "", "", "invalid -remove-trusted value"},
+		{"intrusion-logs", "", "", "", "never", "", "invalid -intrusion-logs value"},
+		{"hash-files", "", "", "", "", "maybe", "invalid -hash-files value"},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			_, err := buildOptions(false, false, tt.backup, tt.download, tt.removeTrusted, tt.intrusionLogs, "")
+			_, err := buildOptions(false, false, tt.backup, tt.download, tt.removeTrusted, tt.intrusionLogs, tt.hashFiles, "")
 			if err == nil || !strings.Contains(err.Error(), tt.wantErr) {
 				t.Fatalf("err = %v, want containing %q", err, tt.wantErr)
 			}
@@ -39,28 +40,28 @@ func TestBuildOptionsInvalidValueFails(t *testing.T) {
 }
 
 func TestBuildOptionsNonInteractiveUnknownModuleFails(t *testing.T) {
-	_, err := buildOptions(false, true, "", "", "", "", "typo")
+	_, err := buildOptions(false, true, "", "", "", "", "", "typo")
 	if err == nil || !strings.Contains(err.Error(), "unknown -module value") {
 		t.Fatalf("err = %v, want unknown -module error", err)
 	}
 }
 
 func TestBuildOptionsNonInteractiveMissingFlagsFails(t *testing.T) {
-	_, err := buildOptions(false, true, "", "", "", "", "")
+	_, err := buildOptions(false, true, "", "", "", "", "", "")
 	if err == nil || !strings.Contains(err.Error(), "-non-interactive requires") {
 		t.Fatalf("err = %v, want missing flags error", err)
 	}
 }
 
 func TestBuildOptionsNonInteractiveModuleFilter(t *testing.T) {
-	_, err := buildOptions(false, true, "none", "", "", "", "backup")
+	_, err := buildOptions(false, true, "none", "", "", "", "", "backup")
 	if err != nil {
 		t.Fatalf("unexpected error: %v", err)
 	}
 }
 
 func TestBuildOptionsFullInvocation(t *testing.T) {
-	opts, err := buildOptions(true, true, "sms", "all", "no", "no", "")
+	opts, err := buildOptions(true, true, "sms", "all", "no", "no", "yes", "")
 	if err != nil {
 		t.Fatalf("unexpected error: %v", err)
 	}
@@ -76,6 +77,7 @@ func TestBuildOptionsFullInvocation(t *testing.T) {
 		{opts.Download, modules.ParseDownloadOption, "all"},
 		{opts.RemoveTrusted, modules.ParseRemoveTrustedOption, "no"},
 		{opts.IntrusionLogs, modules.ParseIntrusionLogsOption, "no"},
+		{opts.HashFiles, modules.ParseHashFilesOption, "yes"},
 	} {
 		want, err := check.parse(check.token)
 		if err != nil {

modules/files.go

@@ -5,14 +5,28 @@
 package modules
 
 import (
+	"fmt"
+	"strings"
+
 	"github.com/botherder/go-savetime/slice"
+	"github.com/manifoldco/promptui"
 	"github.com/mvt-project/androidqf/acquisition"
 	"github.com/mvt-project/androidqf/adb"
 	"github.com/mvt-project/androidqf/log"
 )
 
+const (
+	hashFiles  = "Yes"
+	skipHashes = "No"
+)
+
 type Files struct{}
 
+type fileFinder interface {
+	Find(path string) ([]adb.FileInfo, error)
+	FindHash(path string) ([]adb.FileInfo, error)
+}
+
 func NewFiles() *Files {
 	return &Files{}
 }
@@ -21,7 +35,37 @@ func (f *Files) Name() string {
 	return "files"
 }
 
+func ParseHashFilesOption(value string) (string, error) {
+	switch strings.ToLower(strings.TrimSpace(value)) {
+	case "yes":
+		return hashFiles, nil
+	case "no":
+		return skipHashes, nil
+	}
+	return "", fmt.Errorf("invalid -hash-files value %q (valid values: yes, no)", value)
+}
+
+func findFiles(collector fileFinder, path string, withHashes bool) ([]adb.FileInfo, error) {
+	if withHashes {
+		return collector.FindHash(path)
+	}
+	return collector.Find(path)
+}
+
 func (f *Files) Run(acq *acquisition.Acquisition, opts *Options) error {
+	hashOption, err := resolveOption(opts, opts.HashFiles, "-hash-files (yes, no)", func() (string, error) {
+		log.Info("Would you like to hash files on the device? This is resource-intensive and may cause the collector to stop on some devices.")
+		promptHash := promptui.Select{
+			Label: "Hash files",
+			Items: []string{skipHashes, hashFiles},
+		}
+		_, selection, err := promptHash.Run()
+		return selection, err
+	})
+	if err != nil {
+		return fmt.Errorf("failed to make selection for file hashing option: %v", err)
+	}
+
 	log.Info("Collecting list of files... This might take a while...")
 	var fileFounds []string
 	var fileDetails []adb.FileInfo
@@ -39,6 +83,9 @@ func (f *Files) Run(acq *acquisition.Acquisition, opts *Options) error {
 	} else {
 		log.Debug("Using collector to collect list of files")
 	}
+	if hashOption == hashFiles && method != "collector" {
+		log.Warning("File hashing requires the collector, which is unavailable. Continuing without file hashes.")
+	}
 
 	folders := []string{
 		"/sdcard/", "/system/", "/system_ext/", "/vendor/",
@@ -57,7 +104,7 @@ func (f *Files) Run(acq *acquisition.Acquisition, opts *Options) error {
 		var out []adb.FileInfo
 		var err error
 		if method == "collector" {
-			out, err = acq.Collector.Find(folder)
+			out, err = findFiles(acq.Collector, folder, hashOption == hashFiles)
 		} else if method == "findfull" {
 			out, err = adb.Client.FindFullCommand(folder)
 		} else {

modules/files_test.go

@@ -0,0 +1,40 @@
+package modules
+
+import (
+	"testing"
+
+	"github.com/mvt-project/androidqf/adb"
+)
+
+type recordingFileFinder struct {
+	findCalls     []string
+	findHashCalls []string
+}
+
+func (f *recordingFileFinder) Find(path string) ([]adb.FileInfo, error) {
+	f.findCalls = append(f.findCalls, path)
+	return nil, nil
+}
+
+func (f *recordingFileFinder) FindHash(path string) ([]adb.FileInfo, error) {
+	f.findHashCalls = append(f.findHashCalls, path)
+	return nil, nil
+}
+
+func TestFindFilesUsesRequestedCollectorMode(t *testing.T) {
+	finder := &recordingFileFinder{}
+
+	if _, err := findFiles(finder, "/without-hashes", false); err != nil {
+		t.Fatalf("find without hashes: %v", err)
+	}
+	if _, err := findFiles(finder, "/with-hashes", true); err != nil {
+		t.Fatalf("find with hashes: %v", err)
+	}
+
+	if len(finder.findCalls) != 1 || finder.findCalls[0] != "/without-hashes" {
+		t.Fatalf("Find calls = %v, want [/without-hashes]", finder.findCalls)
+	}
+	if len(finder.findHashCalls) != 1 || finder.findHashCalls[0] != "/with-hashes" {
+		t.Fatalf("FindHash calls = %v, want [/with-hashes]", finder.findHashCalls)
+	}
+}

modules/options.go

@@ -20,6 +20,7 @@ type Options struct {
 	Download       string
 	RemoveTrusted  string
 	IntrusionLogs  string
+	HashFiles      string
 }
 
 func ModuleEnabled(name, filter string) bool {
@@ -58,6 +59,9 @@ func ValidateNonInteractive(opts *Options, moduleFilter string) error {
 	if ModuleEnabled(NewIL().Name(), moduleFilter) && opts.IntrusionLogs == "" {
 		missing = append(missing, "-intrusion-logs (yes, no)")
 	}
+	if ModuleEnabled(NewFiles().Name(), moduleFilter) && opts.HashFiles == "" {
+		missing = append(missing, "-hash-files (yes, no)")
+	}
 
 	if len(missing) == 0 {
 		return nil

modules/options_test.go

@@ -28,6 +28,9 @@ func TestParseOptions(t *testing.T) {
 		{"intrusion-logs yes", ParseIntrusionLogsOption, "yes", acquireIL, ""},
 		{"intrusion-logs no", ParseIntrusionLogsOption, "no", skipIL, ""},
 		{"intrusion-logs invalid", ParseIntrusionLogsOption, "never", "", "invalid -intrusion-logs value"},
+		{"hash-files yes", ParseHashFilesOption, "yes", hashFiles, ""},
+		{"hash-files no", ParseHashFilesOption, "no", skipHashes, ""},
+		{"hash-files invalid", ParseHashFilesOption, "maybe", "", "invalid -hash-files value"},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
@@ -124,7 +127,7 @@ func TestValidateNonInteractive(t *testing.T) {
 			"nothing set",
 			&Options{NonInteractive: true},
 			"",
-			[]string{"-backup", "-download", "-intrusion-logs"},
+			[]string{"-backup", "-download", "-intrusion-logs", "-hash-files"},
 			[]string{"-remove-trusted"},
 		},
 		{
@@ -136,7 +139,7 @@ func TestValidateNonInteractive(t *testing.T) {
 		},
 		{
 			"download none skips remove-trusted",
-			&Options{NonInteractive: true, Backup: backupNothing, Download: apkNone, IntrusionLogs: skipIL},
+			&Options{NonInteractive: true, Backup: backupNothing, Download: apkNone, IntrusionLogs: skipIL, HashFiles: skipHashes},
 			"",
 			nil,
 			nil,
@@ -146,11 +149,25 @@ func TestValidateNonInteractive(t *testing.T) {
 			&Options{NonInteractive: true},
 			"backup",
 			[]string{"-backup"},
-			[]string{"-download", "-intrusion-logs"},
+			[]string{"-download", "-intrusion-logs", "-hash-files"},
+		},
+		{
+			"files module requires hash choice",
+			&Options{NonInteractive: true},
+			"files",
+			[]string{"-hash-files"},
+			[]string{"-backup", "-download", "-intrusion-logs"},
+		},
+		{
+			"files module accepts hash choice",
+			&Options{NonInteractive: true, HashFiles: skipHashes},
+			"files",
+			nil,
+			nil,
 		},
 		{
 			"all set",
-			&Options{NonInteractive: true, Backup: backupOnlySMS, Download: apkAll, RemoveTrusted: apkKeepAll, IntrusionLogs: skipIL},
+			&Options{NonInteractive: true, Backup: backupOnlySMS, Download: apkAll, RemoveTrusted: apkKeepAll, IntrusionLogs: skipIL, HashFiles: hashFiles},
 			"",
 			nil,
 			nil,