google takeout improvements (#409)

* errors when uploading are disturbing the the % of the progression Fixes #376 * add test for Question: report shows way less images uploaded than scanned #390 * add tests for #390 * add fakefs to test with takeout lists * wip: duplicate count * wip: counters * WIP takeout by directory * fix name matchers for duplicate in year and live photo * wip fixe: missing image from the same directory but different type * edit release.md * fix .MP.jpg * fix: Problem with images with same name #402 * fix Wrong creation date results in false album assignment #392 * fix Problem with images with same name #402 , #390, #376, #401 * fix tests * Merge branch 'main' into append-zip-list-fake-file-system * linter fix * linter fix
simulot · Jul 29, 2024 · 6db4ce2 · 6db4ce2
1 parent 3510bf2
commit 6db4ce2
Show file tree

Hide file tree

Showing 27 changed files with 2,783 additions and 414 deletions.
diff --git a/browser/gp/googlephotos.go b/browser/gp/googlephotos.go
diff --git a/browser/gp/googlephotos_test.go b/browser/gp/googlephotos_test.go
@@ -6,37 +6,94 @@ import (
 	"github.com/simulot/immich-go/immich"
 )
 
-func Test_matchEditedName(t *testing.T) {
+func Test_matchers(t *testing.T) {
 	tests := []struct {
 		jsonName string
 		fileName string
-		want     bool
+		want     string
 	}{
+		{
+			jsonName: "PXL_20211013_220651983.jpg.json",
+			fileName: "PXL_20211013_220651983.jpg",
+			want:     "normalMatch",
+		},
 		{
 			jsonName: "PXL_20220405_090123740.PORTRAIT.jpg.json",
 			fileName: "PXL_20220405_090123740.PORTRAIT-modifié.jpg",
-			want:     true,
+			want:     "matchEditedName",
 		},
 		{
 			jsonName: "PXL_20220405_090123740.PORTRAIT.jpg.json",
 			fileName: "PXL_20220405_100123740.PORTRAIT-modifié.jpg",
-			want:     false,
+			want:     "",
 		},
 		{
 			jsonName: "DSC_0238.JPG.json",
 			fileName: "DSC_0238.JPG",
-			want:     true,
+			want:     "normalMatch",
+		},
+		{
+			jsonName: "DSC_0238.JPG(1).json",
+			fileName: "DSC_0238(1).JPG",
+			want:     "matchDuplicateInYear",
+		},
+		{
+			jsonName: "IMG_2710.HEIC(1).json",
+			fileName: "IMG_2710(1).HEIC",
+			want:     "matchDuplicateInYear",
+		},
+		{
+			jsonName: "PXL_20231118_035751175.MP.jpg.json",
+			fileName: "PXL_20231118_035751175.MP.jpg",
+			want:     "normalMatch",
+		},
+		{
+			jsonName: "PXL_20231118_035751175.MP.jpg.json",
+			fileName: "PXL_20231118_035751175.MP",
+			want:     "livePhotoMatch",
+		},
+		{
+			jsonName: "PXL_20230809_203449253.LONG_EXPOSURE-02.ORIGIN.json",
+			fileName: "PXL_20230809_203449253.LONG_EXPOSURE-02.ORIGINA.jpg",
+			want:     "matchWithOneCharOmitted",
+		},
+		{
+			jsonName: "05yqt21kruxwwlhhgrwrdyb6chhwszi9bqmzu16w0 2.jp.json",
+			fileName: "05yqt21kruxwwlhhgrwrdyb6chhwszi9bqmzu16w0 2.jpg",
+			want:     "livePhotoMatch",
+		},
+		{
+			jsonName: "😀😃😄😁😆😅😂🤣🥲☺️😊😇🙂🙃😉😌😍🥰😘😗😙😚😋.json",
+			fileName: "😀😃😄😁😆😅😂🤣🥲☺️😊😇🙂🙃😉😌😍🥰😘😗😙😚😋😛.jpg",
+			want:     "matchWithOneCharOmitted",
+		},
+		{
+			jsonName: "Backyard_ceremony_wedding_photography_xxxxxxx_(494).json",
+			fileName: "Backyard_ceremony_wedding_photography_xxxxxxx_m(494).jpg",
+			want:     "matchVeryLongNameWithNumber",
+		},
+		{
+			jsonName: "original_1d4caa6f-16c6-4c3d-901b-9387de10e528_.json",
+			fileName: "original_1d4caa6f-16c6-4c3d-901b-9387de10e528_P.jpg",
+			want:     "matchWithOneCharOmitted",
+		},
+		{
+			jsonName: "original_1d4caa6f-16c6-4c3d-901b-9387de10e528_.json",
+			fileName: "original_1d4caa6f-16c6-4c3d-901b-9387de10e528_P(1).jpg",
+			want:     "matchForgottenDuplicates",
 		},
-		// {
-		// 	jsonName: "DSC_0238.JPG.json",
-		// 	fileName: "DSC_0238(1).JPG",
-		// 	want:     false,
-		// },
 	}
 	for _, tt := range tests {
 		t.Run(tt.fileName, func(t *testing.T) {
-			if got := matchEditedName(tt.jsonName, tt.fileName, immich.DefaultSupportedMedia); got != tt.want {
-				t.Errorf("matchEditedName() = %v, want %v", got, tt.want)
+			matcher := ""
+			for _, m := range matchers {
+				if m.fn(tt.jsonName, tt.fileName, immich.DefaultSupportedMedia) {
+					matcher = m.name
+					break
+				}
+			}
+			if matcher != tt.want {
+				t.Errorf("matcher is '%s', want %v", matcher, tt.want)
 			}
 		})
 	}
@@ -125,3 +182,37 @@ func Test_matchForgottenDuplicates(t *testing.T) {
 		})
 	}
 }
+
+/* indexes, but false
+goos: linux
+goarch: amd64
+pkg: github.com/simulot/immich-go/browser/gp
+cpu: Intel(R) Core(TM) i7-10750H CPU @ 2.60GHz
+Benchmark_matchDuplicateInYear-12    	27067428	        52.06 ns/op	       0 B/op	       0 allocs/op
+PASS
+ok  	github.com/simulot/immich-go/browser/gp	1.458s
+
+goos: linux
+goarch: amd64
+pkg: github.com/simulot/immich-go/browser/gp
+cpu: Intel(R) Core(TM) i7-10750H CPU @ 2.60GHz
+Benchmark_matchDuplicateInYear-12    	  881652	      1491 ns/op	     240 B/op	       4 allocs/op
+PASS
+ok  	github.com/simulot/immich-go/browser/gp	1.332s
+
+
+goos: linux
+goarch: amd64
+pkg: github.com/simulot/immich-go/browser/gp
+cpu: Intel(R) Core(TM) i7-10750H CPU @ 2.60GHz
+Benchmark_matchDuplicateInYear-12    	25737067	        43.88 ns/op	       0 B/op	       0 allocs/op
+PASS
+ok  	github.com/simulot/immich-go/browser/gp	1.180s
+
+*/
+
+func Benchmark_matchDuplicateInYear(b *testing.B) {
+	for i := 0; i < b.N; i++ {
+		matchDuplicateInYear("IMG_3479.JPG(2).json", "IMG_3479(2).JPG", nil)
+	}
+}
diff --git a/browser/gp/testgp_bigread_test.go b/browser/gp/testgp_bigread_test.go
@@ -31,7 +31,7 @@ func TestReadBigTakeout(t *testing.T) {
 		return
 	}
 	cnt := 0
-	fsyss, err := fshelper.ParsePath(m, true)
+	fsyss, err := fshelper.ParsePath(m)
 	to, err := NewTakeout(context.Background(), j, immich.DefaultSupportedMedia, fsyss...)
 	if err != nil {
 		t.Error(err)