From e673534fb3447b5e0dc552215c132ec8395d1afc Mon Sep 17 00:00:00 2001 From: Roman Babenko Date: Tue, 3 Sep 2024 18:03:04 +0300 Subject: [PATCH 01/49] fix wrong F --- meta/288eaba8.csv | 2 +- meta/2e6b3af5.csv | 2 +- meta/387016a6.csv | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/meta/288eaba8.csv b/meta/288eaba8.csv index 71fd736a..b53da45f 100644 --- a/meta/288eaba8.csv +++ b/meta/288eaba8.csv @@ -22,7 +22,7 @@ Id,FileID,Domain,RepoName,FilePath,LineStart,LineEnd,GroundTruth,WithWords,Value 29927,cb05e4cb,GitHub,288eaba8,data/288eaba8/test/cb05e4cb.js,62,62,T,T,17,29,F,F,CharsOnly,,,Secret,3.02,12,F,F,F,Password 29932,d2417b18,GitHub,288eaba8,data/288eaba8/test/d2417b18.js,11,11,T,T,17,38,F,F,Any,,,Secret,3.22,21,F,F,F,Password 30690,e370d734,GitHub,288eaba8,data/288eaba8/test/e370d734.js,141,141,T,T,24,37,F,F,Any,,,Secret,1.71,13,F,F,F,Password -30875,e370d734,GitHub,288eaba8,data/288eaba8/test/e370d734.js,149,149,T,F,64,71,F,F,,,,,0.0,0,F,F,F,Password +30875,e370d734,GitHub,288eaba8,data/288eaba8/test/e370d734.js,149,149,T,F,64,74,F,F,,,,,0.0,0,F,F,F,Password 31159,0d52ab5c,GitHub,288eaba8,data/288eaba8/src/0d52ab5c.yml,74,74,Template,T,26,34,F,F,CharsOnly,,,Secret,2.5,8,F,F,F,Password 31168,0d52ab5c,GitHub,288eaba8,data/288eaba8/src/0d52ab5c.yml,34,34,Template,T,28,44,F,F,CharsOnly,,,Secret,2.98,16,F,F,F,Password 31170,0d52ab5c,GitHub,288eaba8,data/288eaba8/src/0d52ab5c.yml,37,37,T,T,23,35,F,F,CharsOnly,,,Secret,3.02,12,F,F,F,Password diff --git a/meta/2e6b3af5.csv b/meta/2e6b3af5.csv index acb80c8c..b53c2e7a 100644 --- a/meta/2e6b3af5.csv +++ b/meta/2e6b3af5.csv @@ -96,7 +96,7 @@ Id,FileID,Domain,RepoName,FilePath,LineStart,LineEnd,GroundTruth,WithWords,Value 101828,55134d12,GitHub,2e6b3af5,data/2e6b3af5/test/55134d12.java,28,28,F,F,,,F,F,,,,,0.00,,F,F,F,Auth 101829,55134d12,GitHub,2e6b3af5,data/2e6b3af5/test/55134d12.java,52,52,F,F,,,F,F,,,,,0.00,,F,F,F,Auth 110953,da6cca9c,GitHub,2e6b3af5,data/2e6b3af5/src/da6cca9c.java,15,15,F,F,,,F,F,,,,,0.00,,F,F,F,Nonce -110954,d13e91d9,GitHub,2e6b3af5,data/2e6b3af5/test/d13e91d9.java,110,110,F,F,,,F,F,,,,,0.00,,F,F,F,Token +110954,d13e91d9,GitHub,2e6b3af5,data/2e6b3af5/test/d13e91d9.java,110,110,T,F,58,86,F,F,,,,,0.00,,F,F,F,Token 110955,9e9ce731,GitHub,2e6b3af5,data/2e6b3af5/test/9e9ce731.java,28,28,F,F,,,F,F,,,,,0.00,,F,F,F,Auth 110956,caf3e92f,GitHub,2e6b3af5,data/2e6b3af5/test/caf3e92f.java,42,42,F,F,,,F,F,,,,,0.00,,F,F,F,Token 110957,20f14c80,GitHub,2e6b3af5,data/2e6b3af5/test/20f14c80.java,35,35,F,F,-1,-1,F,F,,,,,0.0,-1,F,F,F,Key diff --git a/meta/387016a6.csv b/meta/387016a6.csv index 333b1493..96dba39d 100644 --- a/meta/387016a6.csv +++ b/meta/387016a6.csv @@ -922,8 +922,8 @@ Id,FileID,Domain,RepoName,FilePath,LineStart,LineEnd,GroundTruth,WithWords,Value 61997,b8da60f9,GitHub,387016a6,data/387016a6/test/b8da60f9.java,621,621,F,F,-1,-1,F,F,,,,,0.0,0,F,F,F,Key 61998,b8da60f9,GitHub,387016a6,data/387016a6/test/b8da60f9.java,581,581,F,F,-1,-1,F,F,,,,,0.0,0,F,F,F,Key 61999,b8da60f9,GitHub,387016a6,data/387016a6/test/b8da60f9.java,615,615,F,F,-1,-1,F,F,,,,,0.0,0,F,F,F,Key -62002,73cd6bff,GitHub,387016a6,data/387016a6/test/73cd6bff.java,213,213,F,F,,,F,F,,,,,0,0,F,F,F,Password -62003,73cd6bff,GitHub,387016a6,data/387016a6/test/73cd6bff.java,141,141,F,F,,,F,F,,,,,0,0,F,F,F,Password +62002,73cd6bff,GitHub,387016a6,data/387016a6/test/73cd6bff.java,213,213,T,F,27,44,F,F,,,,,0,0,F,F,F,Password +62003,73cd6bff,GitHub,387016a6,data/387016a6/test/73cd6bff.java,141,141,T,F,27,38,F,F,,,,,0,0,F,F,F,Password 62004,eedec1c5,GitHub,387016a6,data/387016a6/test/eedec1c5.java,245,245,F,F,,,F,F,,,,,0,0,F,F,F,Password 62009,2125c66c,GitHub,387016a6,data/387016a6/test/2125c66c.java,96,96,F,F,,,F,F,,,,,0,0,F,F,F,Token 62010,efcc3a68,GitHub,387016a6,data/387016a6/test/efcc3a68.java,90,90,F,F,,,F,F,,,,,0,0,F,F,F,Token From 9aa5ca91a6961ef9e0bba69bc7e556f6a25853b3 Mon Sep 17 00:00:00 2001 From: Roman Babenko Date: Tue, 3 Sep 2024 18:21:33 +0300 Subject: [PATCH 02/49] BM upd --- .ci/README.md | 11 +++++++++++ .ci/benchmark.txt | 14 +++++++------- 2 files changed, 18 insertions(+), 7 deletions(-) create mode 100644 .ci/README.md diff --git a/.ci/README.md b/.ci/README.md new file mode 100644 index 00000000..cfdb39b2 --- /dev/null +++ b/.ci/README.md @@ -0,0 +1,11 @@ +# Directory to store auxiliary CI/CD files + +The following files are used for: + * **benchmark.txt** - template scores to compare it with empty result + +To update the benchmark file, do: + +```.venv/bin/python -m benchmark --scanner credsweeper --load .ci/empty_report.json | tee .ci/benchmark.txt``` + + + * **empty_report.json** - empty report from CredSweeper diff --git a/.ci/benchmark.txt b/.ci/benchmark.txt index c19e4fe7..8b8b837f 100644 --- a/.ci/benchmark.txt +++ b/.ci/benchmark.txt @@ -1,5 +1,5 @@ -META MD5 09ddf796180413981211440a025cfb27 -DATA MD5 7b62b847e5b876e1418ad2fc26d2ed35 +META MD5 514244ee1914d923ac3b4db607edb764 +DATA MD5 2fe1d6cff9dd73fe75e961c10871840d DATA: 16345596 interested lines. MARKUP: 62804 items FileType FileNumber ValidLines Positives Negatives Templates --------------- ------------ ------------ ----------- ----------- ----------- @@ -82,7 +82,7 @@ FileType FileNumber ValidLines Positives Negatives Templat .ipynb 1 134 5 .j 1 241 4 .j2 30 5530 6 186 10 -.java 621 134132 361 1367 170 +.java 621 134132 364 1364 170 .jenkinsfile 1 58 2 6 .jinja2 1 64 2 .js 659 536413 535 2496 328 @@ -223,7 +223,7 @@ FileType FileNumber ValidLines Positives Negatives Templat .yml 419 36169 548 891 379 .zsh 6 872 12 .zsh-theme 1 97 1 -TOTAL: 10264 16345596 12186 50511 5103 +TOTAL: 10264 16345596 12189 50508 5103 credsweeper result_cnt : 0, lost_cnt : 0, true_cnt : 0, false_cnt : 0 Rules Positives Negatives Templates Reported TP FP TN FN FPR FNR ACC PRC RCL F1 ------------------------------ ----------- ----------- ----------- ---------- ---- ---- ----- ----- -------- -------- -------- ----- -------- ---- @@ -262,13 +262,13 @@ Key 3899 15718 482 Nonce 91 49 0 0 0 49 91 0.000000 1.000000 0.350000 0.000000 Other 0 8291 1 0 0 8292 0 0.000000 1.000000 PEM Private Key 1019 1483 0 0 0 1483 1019 0.000000 1.000000 0.592726 0.000000 -Password 1854 7542 2702 0 0 10244 1854 0.000000 1.000000 0.846752 0.000000 +Password 1856 7540 2702 0 0 10242 1856 0.000000 1.000000 0.846586 0.000000 Salt 45 76 2 0 0 78 45 0.000000 1.000000 0.634146 0.000000 Secret 1297 1575 799 0 0 2374 1297 0.000000 1.000000 0.646690 0.000000 Seed 1 6 0 0 0 6 1 0.000000 1.000000 0.857143 0.000000 Slack Token 4 1 0 0 0 1 4 0.000000 1.000000 0.200000 0.000000 -Token 646 4176 438 0 0 4614 646 0.000000 1.000000 0.877186 0.000000 +Token 647 4175 438 0 0 4613 647 0.000000 1.000000 0.876996 0.000000 Twilio API Key 0 5 2 0 0 7 0 0.000000 1.000000 URL Credentials 210 151 220 0 0 371 210 0.000000 1.000000 0.638554 0.000000 UUID 1069 265 0 0 0 265 1069 0.000000 1.000000 0.198651 0.000000 - 12186 50511 5103 0 0 0 50511 12186 0.000000 1.000000 0.805637 0.000000 + 12189 50508 5103 0 0 0 50508 12189 0.000000 1.000000 0.805589 0.000000 From 6d45a65b5257ffe652e0ac2c238ed653d7aeabf0 Mon Sep 17 00:00:00 2001 From: Roman Babenko Date: Wed, 4 Sep 2024 13:29:50 +0300 Subject: [PATCH 03/49] avoid CMD ConvertTo-SecureString and password overlap --- meta/387016a6.csv | 2 +- meta/88a0011f.csv | 2 +- meta/a09d9e50.csv | 2 +- meta/a0cd6261.csv | 16 ++++++++-------- 4 files changed, 11 insertions(+), 11 deletions(-) diff --git a/meta/387016a6.csv b/meta/387016a6.csv index 96dba39d..b3aa012a 100644 --- a/meta/387016a6.csv +++ b/meta/387016a6.csv @@ -58,7 +58,7 @@ Id,FileID,Domain,RepoName,FilePath,LineStart,LineEnd,GroundTruth,WithWords,Value 14623,ce05eafb,GitHub,387016a6,data/387016a6/src/ce05eafb.java,56,56,F,F,-1,-1,F,F,,,,,0.0,0,F,F,F,Other 14627,ce05eafb,GitHub,387016a6,data/387016a6/src/ce05eafb.java,66,66,F,F,-1,-1,F,F,,,,,0.0,0,F,F,F,Other 14628,ce05eafb,GitHub,387016a6,data/387016a6/src/ce05eafb.java,58,58,F,F,-1,-1,F,F,,,,,0.0,0,F,F,F,Other -14860,1593006e,GitHub,387016a6,data/387016a6/test/1593006e.java,308,308,T,F,82,89,F,F,,,,,0.0,0,F,F,F,Password:CMD ConvertTo-SecureString +14860,1593006e,GitHub,387016a6,data/387016a6/test/1593006e.java,308,308,T,F,82,89,F,F,,,,,0.0,0,F,F,F,CMD ConvertTo-SecureString 17157,01b42f5f,GitHub,387016a6,data/387016a6/src/01b42f5f.gradle,434,434,F,F,,,F,F,,,,,0,0,F,F,F,API 17451,5da1ec9e,GitHub,387016a6,data/387016a6/src/5da1ec9e.asciidoc,2,2,F,F,,,F,F,,,,,0,0,F,F,F,API 17452,481a72f9,GitHub,387016a6,data/387016a6/src/481a72f9.asciidoc,2,2,F,F,,,F,F,,,,,0,0,F,F,F,API diff --git a/meta/88a0011f.csv b/meta/88a0011f.csv index 122c6a29..2dff7e58 100644 --- a/meta/88a0011f.csv +++ b/meta/88a0011f.csv @@ -1,7 +1,7 @@ Id,FileID,Domain,RepoName,FilePath,LineStart,LineEnd,GroundTruth,WithWords,ValueStart,ValueEnd,InURL,InRuntimeParameter,CharacterSet,CryptographyKey,PredefinedPattern,VariableNameType,Entropy,Length,Base64Encode,HexEncode,URLEncode,Category 109508,b3356305,GitHub,88a0011f,data/88a0011f/other/b3356305.md,1088,1092,F,F,,,F,F,,,,,4.192728,,F,F,F,PEM Private Key 1005284,b3356305,GitHub,88a0011f,data/88a0011f/other/b3356305.md,878,878,T,F,27,36,F,F,,,,,0,0,F,F,F,Password -1113247,b3356305,GitHub,88a0011f,data/88a0011f/other/b3356305.md,395,395,F,F,39,46,F,F,,,,,0.0,0,F,F,F,Password:CMD ConvertTo-SecureString +1113247,b3356305,GitHub,88a0011f,data/88a0011f/other/b3356305.md,395,395,F,F,39,46,F,F,,,,,0.0,0,F,F,F,CMD ConvertTo-SecureString 1479400,b3356305,GitHub,88a0011f,data/88a0011f/other/b3356305.md,312,312,F,F,40,50,F,F,,,,,0.0,0,F,F,F,CMD Password 1479401,b3356305,GitHub,88a0011f,data/88a0011f/other/b3356305.md,317,317,T,F,48,60,F,F,,,,,0.0,0,F,F,F,CMD Password 1479402,b3356305,GitHub,88a0011f,data/88a0011f/other/b3356305.md,326,326,T,F,48,60,F,F,,,,,0.0,0,F,F,F,CMD Password diff --git a/meta/a09d9e50.csv b/meta/a09d9e50.csv index a4752745..4819e1ab 100644 --- a/meta/a09d9e50.csv +++ b/meta/a09d9e50.csv @@ -263,7 +263,7 @@ Id,FileID,Domain,RepoName,FilePath,LineStart,LineEnd,GroundTruth,WithWords,Value 132929,054074fe,GitHub,a09d9e50,data/a09d9e50/test/054074fe.ndjson,3232,3232,T,F,962,972,F,F,,,,,0,0,F,F,F,Password 132930,054074fe,GitHub,a09d9e50,data/a09d9e50/test/054074fe.ndjson,3511,3511,F,F,,,F,F,,,,,0,0,F,F,F,Password 132931,054074fe,GitHub,a09d9e50,data/a09d9e50/test/054074fe.ndjson,3554,3554,F,F,,,F,F,,,,,0,0,F,F,F,Password:API -132932,054074fe,GitHub,a09d9e50,data/a09d9e50/test/054074fe.ndjson,3580,3580,T,F,3017,3028,F,F,,,,,0.0,0,F,F,F,Password:CMD ConvertTo-SecureString +132932,054074fe,GitHub,a09d9e50,data/a09d9e50/test/054074fe.ndjson,3580,3580,T,F,3017,3028,F,F,,,,,0.0,0,F,F,F,CMD ConvertTo-SecureString 132933,054074fe,GitHub,a09d9e50,data/a09d9e50/test/054074fe.ndjson,4342,4342,F,F,,,F,F,,,,,0,0,F,F,F,Password 132934,054074fe,GitHub,a09d9e50,data/a09d9e50/test/054074fe.ndjson,4593,4593,F,F,,,F,F,,,,,0,0,F,F,F,Password 132935,054074fe,GitHub,a09d9e50,data/a09d9e50/test/054074fe.ndjson,4749,4749,F,F,,,F,F,,,,,0,0,F,F,F,Password diff --git a/meta/a0cd6261.csv b/meta/a0cd6261.csv index f4600a76..8059c132 100644 --- a/meta/a0cd6261.csv +++ b/meta/a0cd6261.csv @@ -92,19 +92,19 @@ Id,FileID,Domain,RepoName,FilePath,LineStart,LineEnd,GroundTruth,WithWords,Value 25314,d4acce9c,GitHub,a0cd6261,data/a0cd6261/other/d4acce9c.md,38,38,Template,T,56,67,F,F,Any,,,Secret,2.15,11,F,F,F,Password 25452,b0b094af,GitHub,a0cd6261,data/a0cd6261/other/b0b094af.md,511,511,F,F,-1,-1,F,F,,,,,0.0,0,F,F,F,Other 25921,e955e6ae,GitHub,a0cd6261,data/a0cd6261/other/e955e6ae.md,445,445,F,F,-1,-1,F,F,,,,,0.0,0,F,F,F,Key:Token -26395,919b979a,GitHub,a0cd6261,data/a0cd6261/other/919b979a.md,185,185,T,F,36,55,F,F,,,,,0.0,0,F,F,F,Password:CMD ConvertTo-SecureString +26395,919b979a,GitHub,a0cd6261,data/a0cd6261/other/919b979a.md,185,185,T,F,36,55,F,F,,,,,0.0,0,F,F,F,CMD ConvertTo-SecureString 26630,92025475,GitHub,a0cd6261,data/a0cd6261/other/92025475.md,262,262,F,F,,,F,F,,,,,0,0,F,F,F,Token 28831,3b98604a,GitHub,a0cd6261,data/a0cd6261/other/3b98604a.md,762,762,T,F,57,68,F,F,Any,,,Secret,2.52,12,F,F,F,Password -29115,3b98604a,GitHub,a0cd6261,data/a0cd6261/other/3b98604a.md,1686,1686,T,T,35,43,F,F,Any,,,Unknown,3.37,23,F,F,F,Password:CMD ConvertTo-SecureString +29115,3b98604a,GitHub,a0cd6261,data/a0cd6261/other/3b98604a.md,1686,1686,T,T,35,43,F,F,Any,,,Unknown,3.37,23,F,F,F,CMD ConvertTo-SecureString 29132,7f6a3252,GitHub,a0cd6261,data/a0cd6261/other/7f6a3252.md,96,96,Template,T,30,45,F,F,CharsOnly,,,Token,3.37,15,F,F,F,Token 29142,7f6a3252,GitHub,a0cd6261,data/a0cd6261/other/7f6a3252.md,95,95,Template,T,25,40,F,F,CharsOnly,,,Token,3.37,15,F,F,F,Token 29143,7f6a3252,GitHub,a0cd6261,data/a0cd6261/other/7f6a3252.md,97,97,Template,T,22,37,F,F,CharsOnly,,,Token,3.37,15,F,F,F,Token 29148,e955e6ae,GitHub,a0cd6261,data/a0cd6261/other/e955e6ae.md,846,846,T,T,49,60,F,F,Any,,,Unknown,3.12,11,F,F,F,Password 29150,e955e6ae,GitHub,a0cd6261,data/a0cd6261/other/e955e6ae.md,847,847,T,T,49,60,F,F,Any,,,Unknown,3.12,11,F,F,F,Password 29160,919b979a,GitHub,a0cd6261,data/a0cd6261/other/919b979a.md,80,80,Template,F,10,15,F,F,Any,,,Secret,2.32,5,F,F,F,Password -29165,3b98604a,GitHub,a0cd6261,data/a0cd6261/other/3b98604a.md,1719,1719,T,T,41,49,F,F,Any,,,Secret,3.37,23,F,F,F,Password:CMD ConvertTo-SecureString -29166,3b98604a,GitHub,a0cd6261,data/a0cd6261/other/3b98604a.md,1769,1769,T,T,39,51,T,F,Any,,,Secret,3.37,23,F,F,F,Password:CMD ConvertTo-SecureString -29215,e955e6ae,GitHub,a0cd6261,data/a0cd6261/other/e955e6ae.md,921,921,F,T,-1,-1,F,F,Any,,,Secret,3.37,23,F,F,F,Password:CMD ConvertTo-SecureString +29165,3b98604a,GitHub,a0cd6261,data/a0cd6261/other/3b98604a.md,1719,1719,T,T,41,49,F,F,Any,,,Secret,3.37,23,F,F,F,CMD ConvertTo-SecureString +29166,3b98604a,GitHub,a0cd6261,data/a0cd6261/other/3b98604a.md,1769,1769,T,T,39,51,T,F,Any,,,Secret,3.37,23,F,F,F,CMD ConvertTo-SecureString +29215,e955e6ae,GitHub,a0cd6261,data/a0cd6261/other/e955e6ae.md,921,921,F,T,-1,-1,F,F,Any,,,Secret,3.37,23,F,F,F,CMD ConvertTo-SecureString 29258,b3e1b56b,GitHub,a0cd6261,data/a0cd6261/other/b3e1b56b.md,237,237,T,T,43,57,F,F,Any,,,Secret,2.91,14,F,F,F,Password 29351,b0464027,GitHub,a0cd6261,data/a0cd6261/other/b0464027.md,666,666,T,F,67,94,T,F,Any,,Google OAuth Access Token,Auth,2.99,27,F,F,F,Google OAuth Access Token:Token 29684,a2d793d9,GitHub,a0cd6261,data/a0cd6261/other/a2d793d9.md,215,215,Template,T,48,52,T,F,Any,,,Auth,1.50,4,F,F,F,Auth @@ -112,12 +112,12 @@ Id,FileID,Domain,RepoName,FilePath,LineStart,LineEnd,GroundTruth,WithWords,Value 29851,3b98604a,GitHub,a0cd6261,data/a0cd6261/other/3b98604a.md,2165,2165,T,T,31,42,F,F,Any,,,Secret,2.96,11,F,F,F,Password 30031,29bb2e65,GitHub,a0cd6261,data/a0cd6261/test/29bb2e65.md,254,254,T,F,25,60,F,F,Any,,,Secret,3.93,35,F,F,F,Key:Secret 30085,29bb2e65,GitHub,a0cd6261,data/a0cd6261/test/29bb2e65.md,88,88,T,F,87,123,F,F,Any,,,Secret,4.78,36,F,F,F,Key:Secret -30509,7269210a,GitHub,a0cd6261,data/a0cd6261/test/7269210a.md,201,201,T,T,42,59,F,F,Any,,,Secret,3.37,23,F,F,F,Password:CMD ConvertTo-SecureString +30509,7269210a,GitHub,a0cd6261,data/a0cd6261/test/7269210a.md,201,201,T,T,42,59,F,F,Any,,,Secret,3.37,23,F,F,F,CMD ConvertTo-SecureString 30658,c4cf4dfc,GitHub,a0cd6261,data/a0cd6261/other/c4cf4dfc.md,132,132,Template,T,73,97,T,F,Any,,,Token,3.48,24,F,F,F,Token 30659,c4cf4dfc,GitHub,a0cd6261,data/a0cd6261/other/c4cf4dfc.md,91,91,Template,T,63,82,T,F,Any,,,Token,2.93,19,F,F,F,Token 30660,c4cf4dfc,GitHub,a0cd6261,data/a0cd6261/other/c4cf4dfc.md,67,67,Template,F,73,77,T,F,CharsOnly,,,Secret,0,4,F,F,F,Password 30667,c4cf4dfc,GitHub,a0cd6261,data/a0cd6261/other/c4cf4dfc.md,129,129,Template,T,52,76,T,F,Any,,,Token,3.48,24,F,F,F,Token -30769,919b979a,GitHub,a0cd6261,data/a0cd6261/other/919b979a.md,127,127,T,T,47,61,F,F,Any,,,Secret,3.37,23,F,F,F,Password:CMD ConvertTo-SecureString +30769,919b979a,GitHub,a0cd6261,data/a0cd6261/other/919b979a.md,127,127,T,T,47,61,F,F,Any,,,Secret,3.37,23,F,F,F,CMD ConvertTo-SecureString 30936,3b98604a,GitHub,a0cd6261,data/a0cd6261/other/3b98604a.md,2368,2368,T,T,29,38,F,F,Any,,,Secret,2.95,9,F,F,F,Password 31038,a5bf5de5,GitHub,a0cd6261,data/a0cd6261/other/a5bf5de5.md,84,84,Template,T,53,62,F,F,CharsOnly,,,Secret,2.95,9,F,F,F,Password 31108,3b98604a,GitHub,a0cd6261,data/a0cd6261/other/3b98604a.md,2166,2166,T,T,37,48,F,F,Any,,,Secret,2.96,11,F,F,F,Password @@ -203,7 +203,7 @@ Id,FileID,Domain,RepoName,FilePath,LineStart,LineEnd,GroundTruth,WithWords,Value 114013,d73e7ecc,GitHub,a0cd6261,data/a0cd6261/other/d73e7ecc.md,54,54,F,F,,,F,F,,,,,0.00,,F,F,F,AWS S3 Bucket 114014,b0464027,GitHub,a0cd6261,data/a0cd6261/other/b0464027.md,660,660,F,F,,,F,F,,,,,0.00,,F,F,F,Token 114015,c7caa3f3,GitHub,a0cd6261,data/a0cd6261/other/c7caa3f3.md,211,211,T,F,100,228,F,F,,,,,0.0,-1,F,F,F,Key -114016,3b98604a,GitHub,a0cd6261,data/a0cd6261/other/3b98604a.md,1688,1688,T,F,39,52,F,F,,,,,0.0,-1,F,F,F,Password:CMD ConvertTo-SecureString +114016,3b98604a,GitHub,a0cd6261,data/a0cd6261/other/3b98604a.md,1688,1688,T,F,39,52,F,F,,,,,0.0,-1,F,F,F,CMD ConvertTo-SecureString 114017,3b98604a,GitHub,a0cd6261,data/a0cd6261/other/3b98604a.md,230,230,F,F,,,F,F,,,,,0.00,,F,F,F,Auth 114018,b0b094af,GitHub,a0cd6261,data/a0cd6261/other/b0b094af.md,631,631,F,F,,,F,F,,,,,0.00,,F,F,F,Password 114019,b0b094af,GitHub,a0cd6261,data/a0cd6261/other/b0b094af.md,641,641,F,F,,,F,F,,,,,0.00,,F,F,F,Password From 8ca09cf8ce760454c8f641ec85500d14c63cf0dc Mon Sep 17 00:00:00 2001 From: Roman Babenko Date: Wed, 4 Sep 2024 14:11:50 +0300 Subject: [PATCH 04/49] BMCIFIX --- .ci/benchmark.txt | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.ci/benchmark.txt b/.ci/benchmark.txt index 8b8b837f..01e51f8b 100644 --- a/.ci/benchmark.txt +++ b/.ci/benchmark.txt @@ -1,4 +1,4 @@ -META MD5 514244ee1914d923ac3b4db607edb764 +META MD5 a971a87bd84d1b66f06447bc3a84011c DATA MD5 2fe1d6cff9dd73fe75e961c10871840d DATA: 16345596 interested lines. MARKUP: 62804 items FileType FileNumber ValidLines Positives Negatives Templates @@ -82,7 +82,7 @@ FileType FileNumber ValidLines Positives Negatives Templat .ipynb 1 134 5 .j 1 241 4 .j2 30 5530 6 186 10 -.java 621 134132 364 1364 170 +.java 621 134132 363 1364 170 .jenkinsfile 1 58 2 6 .jinja2 1 64 2 .js 659 536413 535 2496 328 @@ -113,7 +113,7 @@ FileType FileNumber ValidLines Positives Negatives Templat .markdown 3 139 3 1 .markerb 3 12 3 .marko 1 21 2 -.md 675 149568 738 2395 623 +.md 675 149568 731 2393 623 .mdx 3 549 7 .mjml 1 18 1 .mjs 22 4424 75 340 @@ -125,7 +125,7 @@ FileType FileNumber ValidLines Positives Negatives Templat .mqh 1 1023 2 .msg 1 26644 1 1 .mysql 1 36 2 -.ndjson 2 5006 75 239 2 +.ndjson 2 5006 74 239 2 .nix 4 211 12 .nolint 1 2 1 .odd 1 1281 43 @@ -223,7 +223,7 @@ FileType FileNumber ValidLines Positives Negatives Templat .yml 419 36169 548 891 379 .zsh 6 872 12 .zsh-theme 1 97 1 -TOTAL: 10264 16345596 12189 50508 5103 +TOTAL: 10264 16345596 12180 50506 5103 credsweeper result_cnt : 0, lost_cnt : 0, true_cnt : 0, false_cnt : 0 Rules Positives Negatives Templates Reported TP FP TN FN FPR FNR ACC PRC RCL F1 ------------------------------ ----------- ----------- ----------- ---------- ---- ---- ----- ----- -------- -------- -------- ----- -------- ---- @@ -262,7 +262,7 @@ Key 3899 15718 482 Nonce 91 49 0 0 0 49 91 0.000000 1.000000 0.350000 0.000000 Other 0 8291 1 0 0 8292 0 0.000000 1.000000 PEM Private Key 1019 1483 0 0 0 1483 1019 0.000000 1.000000 0.592726 0.000000 -Password 1856 7540 2702 0 0 10242 1856 0.000000 1.000000 0.846586 0.000000 +Password 1847 7538 2702 0 0 10240 1847 0.000000 1.000000 0.847191 0.000000 Salt 45 76 2 0 0 78 45 0.000000 1.000000 0.634146 0.000000 Secret 1297 1575 799 0 0 2374 1297 0.000000 1.000000 0.646690 0.000000 Seed 1 6 0 0 0 6 1 0.000000 1.000000 0.857143 0.000000 @@ -271,4 +271,4 @@ Token 647 4175 438 Twilio API Key 0 5 2 0 0 7 0 0.000000 1.000000 URL Credentials 210 151 220 0 0 371 210 0.000000 1.000000 0.638554 0.000000 UUID 1069 265 0 0 0 265 1069 0.000000 1.000000 0.198651 0.000000 - 12189 50508 5103 0 0 0 50508 12189 0.000000 1.000000 0.805589 0.000000 + 12180 50506 5103 0 0 0 50506 12180 0.000000 1.000000 0.805698 0.000000 From 91ae1d2b5b3c15f6923cc136187f47095b11dc61 Mon Sep 17 00:00:00 2001 From: Roman Babenko Date: Wed, 4 Sep 2024 15:49:42 +0300 Subject: [PATCH 05/49] markup upd --- .ci/benchmark.txt | 24 ++++++++++++------------ meta/0f133e09.csv | 2 +- meta/69d49010.csv | 2 +- meta/90aebe4a.csv | 2 +- meta/92faccc6.csv | 2 +- 5 files changed, 16 insertions(+), 16 deletions(-) diff --git a/.ci/benchmark.txt b/.ci/benchmark.txt index 01e51f8b..515ba5dd 100644 --- a/.ci/benchmark.txt +++ b/.ci/benchmark.txt @@ -1,5 +1,5 @@ -META MD5 a971a87bd84d1b66f06447bc3a84011c -DATA MD5 2fe1d6cff9dd73fe75e961c10871840d +META MD5 8d549ea2322768735a4841f92d66bf15 +DATA MD5 2339e7d62fc2d718caa86bef8553aabe DATA: 16345596 interested lines. MARKUP: 62804 items FileType FileNumber ValidLines Positives Negatives Templates --------------- ------------ ------------ ----------- ----------- ----------- @@ -113,7 +113,7 @@ FileType FileNumber ValidLines Positives Negatives Templat .markdown 3 139 3 1 .markerb 3 12 3 .marko 1 21 2 -.md 675 149568 731 2393 623 +.md 675 149568 732 2392 623 .mdx 3 549 7 .mjml 1 18 1 .mjs 22 4424 75 340 @@ -153,7 +153,7 @@ FileType FileNumber ValidLines Positives Negatives Templat .pug 2 193 2 .purs 1 69 4 .pxd 1 150 5 2 -.py 890 291553 684 3298 724 +.py 890 291553 683 3299 724 .pyi 4 1361 9 .pyp 1 167 1 .pyx 2 1094 23 @@ -209,7 +209,7 @@ FileType FileNumber ValidLines Positives Negatives Templat .ts 583 106730 159 1800 201 .tsx 54 7914 1 114 5 .ttar 1 452 1 -.txt 440 78102 5271 6373 49 +.txt 440 78102 5272 6372 49 .utf8 1 77 2 .vsixmanifest 1 36 1 .vsmdi 1 6 2 @@ -220,10 +220,10 @@ FileType FileNumber ValidLines Positives Negatives Templat .xml 9 689 9 .xsl 1 311 1 .yaml 137 19004 125 345 42 -.yml 419 36169 548 891 379 +.yml 419 36169 549 890 379 .zsh 6 872 12 .zsh-theme 1 97 1 -TOTAL: 10264 16345596 12180 50506 5103 +TOTAL: 10264 16345596 12182 50504 5103 credsweeper result_cnt : 0, lost_cnt : 0, true_cnt : 0, false_cnt : 0 Rules Positives Negatives Templates Reported TP FP TN FN FPR FNR ACC PRC RCL F1 ------------------------------ ----------- ----------- ----------- ---------- ---- ---- ----- ----- -------- -------- -------- ----- -------- ---- @@ -232,7 +232,7 @@ AWS Client ID 167 21 0 AWS Multi 75 16 0 0 0 16 75 0.000000 1.000000 0.175824 0.000000 AWS S3 Bucket 66 24 0 0 0 24 66 0.000000 1.000000 0.266667 0.000000 Atlassian Old PAT token 27 308 3 0 0 311 27 0.000000 1.000000 0.920118 0.000000 -Auth 420 2734 76 0 0 2810 420 0.000000 1.000000 0.869969 0.000000 +Auth 419 2735 76 0 0 2811 419 0.000000 1.000000 0.870279 0.000000 Azure Access Token 19 0 0 0 0 0 19 1.000000 0.000000 0.000000 BASE64 Private Key 7 4 0 0 0 4 7 0.000000 1.000000 0.363636 0.000000 BASE64 encoded PEM Private Key 7 0 0 0 0 0 7 1.000000 0.000000 0.000000 @@ -262,13 +262,13 @@ Key 3899 15718 482 Nonce 91 49 0 0 0 49 91 0.000000 1.000000 0.350000 0.000000 Other 0 8291 1 0 0 8292 0 0.000000 1.000000 PEM Private Key 1019 1483 0 0 0 1483 1019 0.000000 1.000000 0.592726 0.000000 -Password 1847 7538 2702 0 0 10240 1847 0.000000 1.000000 0.847191 0.000000 +Password 1848 7537 2702 0 0 10239 1848 0.000000 1.000000 0.847108 0.000000 Salt 45 76 2 0 0 78 45 0.000000 1.000000 0.634146 0.000000 -Secret 1297 1575 799 0 0 2374 1297 0.000000 1.000000 0.646690 0.000000 +Secret 1298 1574 799 0 0 2373 1298 0.000000 1.000000 0.646418 0.000000 Seed 1 6 0 0 0 6 1 0.000000 1.000000 0.857143 0.000000 Slack Token 4 1 0 0 0 1 4 0.000000 1.000000 0.200000 0.000000 -Token 647 4175 438 0 0 4613 647 0.000000 1.000000 0.876996 0.000000 +Token 648 4174 438 0 0 4612 648 0.000000 1.000000 0.876806 0.000000 Twilio API Key 0 5 2 0 0 7 0 0.000000 1.000000 URL Credentials 210 151 220 0 0 371 210 0.000000 1.000000 0.638554 0.000000 UUID 1069 265 0 0 0 265 1069 0.000000 1.000000 0.198651 0.000000 - 12180 50506 5103 0 0 0 50506 12180 0.000000 1.000000 0.805698 0.000000 + 12182 50504 5103 0 0 0 50504 12182 0.000000 1.000000 0.805666 0.000000 diff --git a/meta/0f133e09.csv b/meta/0f133e09.csv index bb0aa6f7..285aa92a 100644 --- a/meta/0f133e09.csv +++ b/meta/0f133e09.csv @@ -266,7 +266,7 @@ Id,FileID,Domain,RepoName,FilePath,LineStart,LineEnd,GroundTruth,WithWords,Value 9494,f5819a5c,GitHub,0f133e09,data/0f133e09/test/f5819a5c.txt,10,10,F,F,,,F,F,,,,,0,0,F,F,F,Bitbucket Client ID 9495,f8329470,GitHub,0f133e09,data/0f133e09/test/f8329470.txt,21,21,F,F,,,F,F,,,,,0,0,F,F,F,Bitbucket Client ID 9497,28554f3b,GitHub,0f133e09,data/0f133e09/test/28554f3b.txt,21,21,F,F,,,F,F,,,,,0,0,F,F,F,Bitbucket Client ID -9499,5a7f3ab7,GitHub,0f133e09,data/0f133e09/test/5a7f3ab7.txt,54,54,F,F,,,F,F,,,,,0,0,F,F,F,Token +9499,5a7f3ab7,GitHub,0f133e09,data/0f133e09/test/5a7f3ab7.txt,54,54,T,F,256,296,F,F,,,,,0,0,F,F,F,Token 9500,3ff7b45c,GitHub,0f133e09,data/0f133e09/test/3ff7b45c.txt,43,43,F,F,,,F,F,,,,,0,0,F,F,F,Dropbox App secret 9501,3c654d2d,GitHub,0f133e09,data/0f133e09/test/3c654d2d.txt,43,43,F,F,,,F,F,,,,,0,0,F,F,F,Dropbox App secret 9502,7b3e739d,GitHub,0f133e09,data/0f133e09/test/7b3e739d.txt,21,21,F,F,,,F,F,,,,,0,0,F,F,F,Dropbox App secret diff --git a/meta/69d49010.csv b/meta/69d49010.csv index c1390e9c..7e781966 100644 --- a/meta/69d49010.csv +++ b/meta/69d49010.csv @@ -168,4 +168,4 @@ Id,FileID,Domain,RepoName,FilePath,LineStart,LineEnd,GroundTruth,WithWords,Value 1479352,5aad918a,GitHub,69d49010,data/69d49010/test/5aad918a.py,303,303,T,F,35,48,F,F,,,,,0.0,0,F,F,F,Auth 1479354,5aad918a,GitHub,69d49010,data/69d49010/test/5aad918a.py,329,329,T,F,35,48,F,F,,,,,0.0,0,F,F,F,Auth 1479356,5aad918a,GitHub,69d49010,data/69d49010/test/5aad918a.py,356,356,T,F,35,48,F,F,,,,,0.0,0,F,F,F,Auth -1479361,5aad918a,GitHub,69d49010,data/69d49010/test/5aad918a.py,433,433,T,F,40,51,F,F,,,,,0.0,0,F,F,F,Auth +1479361,5aad918a,GitHub,69d49010,data/69d49010/test/5aad918a.py,433,433,F,F,,,F,F,,,,,0.0,0,F,F,F,Auth diff --git a/meta/90aebe4a.csv b/meta/90aebe4a.csv index c4a8f51b..79086a8e 100644 --- a/meta/90aebe4a.csv +++ b/meta/90aebe4a.csv @@ -84,7 +84,7 @@ Id,FileID,Domain,RepoName,FilePath,LineStart,LineEnd,GroundTruth,WithWords,Value 130889,fa8bba7a,GitHub,90aebe4a,data/90aebe4a/test/fa8bba7a.js,167,167,F,F,,,F,F,,,,,0.00,,F,F,F,Password 130890,a48facfc,GitHub,90aebe4a,data/90aebe4a/test/a48facfc.js,32,32,F,F,,,F,F,,,,,0.00,,F,F,F,Password 130893,c6314109,GitHub,90aebe4a,data/90aebe4a/src/c6314109.js,244,244,F,F,,,F,F,,,,,0.00,,F,F,F,Password -130894,6ac3f79f,GitHub,90aebe4a,data/90aebe4a/src/6ac3f79f.yml,28,28,F,F,,,F,F,,,,,0.00,,F,F,F,Password +130894,6ac3f79f,GitHub,90aebe4a,data/90aebe4a/src/6ac3f79f.yml,28,28,T,F,162,172,F,F,,,,,0.00,,F,F,F,Password 130896,38c3f879,GitHub,90aebe4a,data/90aebe4a/test/38c3f879.js,107,107,F,F,,,F,F,,,,,0.00,,F,F,F,Password 130899,f5a78215,GitHub,90aebe4a,data/90aebe4a/src/f5a78215.js,40,40,F,F,,,F,F,,,,,0.00,,F,F,F,Password 130901,13bd9d7a,GitHub,90aebe4a,data/90aebe4a/other/13bd9d7a,48,48,F,F,,,F,F,,,,,0.00,,F,F,F,Password diff --git a/meta/92faccc6.csv b/meta/92faccc6.csv index 208cbf1a..05d369bc 100644 --- a/meta/92faccc6.csv +++ b/meta/92faccc6.csv @@ -76,7 +76,7 @@ Id,FileID,Domain,RepoName,FilePath,LineStart,LineEnd,GroundTruth,WithWords,Value 82147,8ea6006d,GitHub,92faccc6,data/92faccc6/test/8ea6006d.kt,424,424,F,F,9,97,F,F,,,,,0.0,0,F,F,F,Other 82148,8ea6006d,GitHub,92faccc6,data/92faccc6/test/8ea6006d.kt,429,429,F,F,9,97,F,F,,,,,0.0,0,F,F,F,Other 100615,b4d68dc8,GitHub,92faccc6,data/92faccc6/other/b4d68dc8.md,74,74,F,F,,,F,F,,,,,0.00,,F,F,F,Secret -100616,b4d68dc8,GitHub,92faccc6,data/92faccc6/other/b4d68dc8.md,911,911,F,F,,,F,F,,,,,0.00,,F,F,F,Secret +100616,b4d68dc8,GitHub,92faccc6,data/92faccc6/other/b4d68dc8.md,911,911,T,F,46,70,F,F,,,,,0.00,,F,F,F,Secret 100617,c5175610,GitHub,92faccc6,data/92faccc6/other/c5175610.md,39,39,F,F,,,F,F,,,,,0.00,,F,F,F,Secret 100618,80503e10,GitHub,92faccc6,data/92faccc6/src/80503e10.kt,543,543,F,F,-1,-1,F,F,,,,,0.0,-1,F,F,F,Other 100619,e06aed65,GitHub,92faccc6,data/92faccc6/src/e06aed65.kt,60,60,F,F,-1,-1,F,F,,,,,0.0,-1,F,F,F,Other From c82bc3fc1511a1b008c0481324039612206e62e4 Mon Sep 17 00:00:00 2001 From: Roman Babenko Date: Wed, 4 Sep 2024 16:40:44 +0300 Subject: [PATCH 06/49] fix --- .ci/benchmark.txt | 28 ++++++++++++++-------------- meta/2df212a2.csv | 2 +- meta/64c97cb8.csv | 2 +- meta/69d49010.csv | 4 ++-- meta/7c7558cf.csv | 6 +++--- meta/a0cd6261.csv | 2 ++ 6 files changed, 23 insertions(+), 21 deletions(-) diff --git a/.ci/benchmark.txt b/.ci/benchmark.txt index 515ba5dd..67f3578b 100644 --- a/.ci/benchmark.txt +++ b/.ci/benchmark.txt @@ -1,6 +1,6 @@ -META MD5 8d549ea2322768735a4841f92d66bf15 -DATA MD5 2339e7d62fc2d718caa86bef8553aabe -DATA: 16345596 interested lines. MARKUP: 62804 items +META MD5 8288d8a5b3f7edb9ff8f66a75b1fd6db +DATA MD5 8503780bbece7b6a6f1ba38f59897fdb +DATA: 16345596 interested lines. MARKUP: 62805 items FileType FileNumber ValidLines Positives Negatives Templates --------------- ------------ ------------ ----------- ----------- ----------- 194 28318 70 417 91 @@ -30,7 +30,7 @@ FileType FileNumber ValidLines Positives Negatives Templat .cnf 8 858 15 36 16 .coffee 1 585 2 .conf 60 4945 55 64 53 -.config 20 492 16 38 1 +.config 20 492 17 38 1 .cpp 15 5688 2 61 .creds 1 10 1 1 .crlf 1 27 1 @@ -113,7 +113,7 @@ FileType FileNumber ValidLines Positives Negatives Templat .markdown 3 139 3 1 .markerb 3 12 3 .marko 1 21 2 -.md 675 149568 732 2392 623 +.md 675 149568 734 2392 623 .mdx 3 549 7 .mjml 1 18 1 .mjs 22 4424 75 340 @@ -153,7 +153,7 @@ FileType FileNumber ValidLines Positives Negatives Templat .pug 2 193 2 .purs 1 69 4 .pxd 1 150 5 2 -.py 890 291553 683 3299 724 +.py 890 291553 683 3301 723 .pyi 4 1361 9 .pyp 1 167 1 .pyx 2 1094 23 @@ -172,7 +172,7 @@ FileType FileNumber ValidLines Positives Negatives Templat .rs 31 9855 2 234 11 .rsc 1 691 1 .rsp 16 7101 20 10 27 -.rst 86 33980 69 323 68 +.rst 86 33980 70 323 68 .rules 1 6 2 .sample 2 25 3 4 4 .sbt 3 570 5 2 @@ -209,7 +209,7 @@ FileType FileNumber ValidLines Positives Negatives Templat .ts 583 106730 159 1800 201 .tsx 54 7914 1 114 5 .ttar 1 452 1 -.txt 440 78102 5272 6372 49 +.txt 440 78102 5272 6373 49 .utf8 1 77 2 .vsixmanifest 1 36 1 .vsmdi 1 6 2 @@ -220,15 +220,15 @@ FileType FileNumber ValidLines Positives Negatives Templat .xml 9 689 9 .xsl 1 311 1 .yaml 137 19004 125 345 42 -.yml 419 36169 549 890 379 +.yml 419 36169 552 890 379 .zsh 6 872 12 .zsh-theme 1 97 1 -TOTAL: 10264 16345596 12182 50504 5103 +TOTAL: 10264 16345596 12189 50507 5102 credsweeper result_cnt : 0, lost_cnt : 0, true_cnt : 0, false_cnt : 0 Rules Positives Negatives Templates Reported TP FP TN FN FPR FNR ACC PRC RCL F1 ------------------------------ ----------- ----------- ----------- ---------- ---- ---- ----- ----- -------- -------- -------- ----- -------- ---- API 129 3162 189 0 0 3351 129 0.000000 1.000000 0.962931 0.000000 -AWS Client ID 167 21 0 0 0 21 167 0.000000 1.000000 0.111702 0.000000 +AWS Client ID 168 21 0 0 0 21 168 0.000000 1.000000 0.111111 0.000000 AWS Multi 75 16 0 0 0 16 75 0.000000 1.000000 0.175824 0.000000 AWS S3 Bucket 66 24 0 0 0 24 66 0.000000 1.000000 0.266667 0.000000 Atlassian Old PAT token 27 308 3 0 0 311 27 0.000000 1.000000 0.920118 0.000000 @@ -258,17 +258,17 @@ Grafana Provisioned API Key 22 1 0 JSON Web Token 170 61 0 0 0 61 170 0.000000 1.000000 0.264069 0.000000 Jira / Confluence PAT token 0 4 0 0 0 4 0 0.000000 1.000000 Jira 2FA 15 6 1 0 0 7 15 0.000000 1.000000 0.318182 0.000000 -Key 3899 15718 482 0 0 16200 3899 0.000000 1.000000 0.806010 0.000000 +Key 3905 15719 482 0 0 16201 3905 0.000000 1.000000 0.805779 0.000000 Nonce 91 49 0 0 0 49 91 0.000000 1.000000 0.350000 0.000000 Other 0 8291 1 0 0 8292 0 0.000000 1.000000 PEM Private Key 1019 1483 0 0 0 1483 1019 0.000000 1.000000 0.592726 0.000000 Password 1848 7537 2702 0 0 10239 1848 0.000000 1.000000 0.847108 0.000000 Salt 45 76 2 0 0 78 45 0.000000 1.000000 0.634146 0.000000 -Secret 1298 1574 799 0 0 2373 1298 0.000000 1.000000 0.646418 0.000000 +Secret 1298 1576 798 0 0 2374 1298 0.000000 1.000000 0.646514 0.000000 Seed 1 6 0 0 0 6 1 0.000000 1.000000 0.857143 0.000000 Slack Token 4 1 0 0 0 1 4 0.000000 1.000000 0.200000 0.000000 Token 648 4174 438 0 0 4612 648 0.000000 1.000000 0.876806 0.000000 Twilio API Key 0 5 2 0 0 7 0 0.000000 1.000000 URL Credentials 210 151 220 0 0 371 210 0.000000 1.000000 0.638554 0.000000 UUID 1069 265 0 0 0 265 1069 0.000000 1.000000 0.198651 0.000000 - 12182 50504 5103 0 0 0 50504 12182 0.000000 1.000000 0.805666 0.000000 + 12189 50507 5102 0 0 0 50507 12189 0.000000 1.000000 0.805586 0.000000 diff --git a/meta/2df212a2.csv b/meta/2df212a2.csv index 9af53953..6ce9325c 100644 --- a/meta/2df212a2.csv +++ b/meta/2df212a2.csv @@ -61,7 +61,7 @@ Id,FileID,Domain,RepoName,FilePath,LineStart,LineEnd,GroundTruth,WithWords,Value 30545,6ac10689,GitHub,2df212a2,data/2df212a2/src/6ac10689.rst,120,120,Template,F,26,30,F,F,Any,,,Secret,2.0,4,F,F,F,Password:Key:Secret 30547,6ac10689,GitHub,2df212a2,data/2df212a2/src/6ac10689.rst,124,124,Template,F,26,30,F,F,Any,,,Secret,2.0,4,F,F,F,Password:Key:Secret 30548,7824b4fc,GitHub,2df212a2,data/2df212a2/test/7824b4fc.py,28,28,Template,F,62,68,F,F,Any,,,Secret,2.58,6,F,F,F,Credential:Key -31125,6f0a498c,GitHub,2df212a2,data/2df212a2/test/6f0a498c.rst,25,25,T,T,14,34,F,F,Any,,AWS Client ID,Unknown,3.68,20,F,F,F,AWS Client ID:Gitlab Feed Token +31125,6f0a498c,GitHub,2df212a2,data/2df212a2/test/6f0a498c.rst,25,25,T,T,14,34,F,F,Any,,AWS Client ID,Unknown,3.68,20,F,F,F,AWS Client ID:Gitlab Feed Token:Key 32689,7824b4fc,GitHub,2df212a2,data/2df212a2/test/7824b4fc.py,68,68,Template,T,35,40,F,F,CharsOnly,,,Token,2.32,5,F,F,F,Token 32690,6ac10689,GitHub,2df212a2,data/2df212a2/src/6ac10689.rst,62,62,F,F,26,39,F,F,,,,,0.0,0,F,F,F,Token 36742,7824b4fc,GitHub,2df212a2,data/2df212a2/test/7824b4fc.py,93,93,F,F,30,35,F,F,,,,,0.0,0,F,F,F,Token diff --git a/meta/64c97cb8.csv b/meta/64c97cb8.csv index 8f5208c4..1e16594b 100644 --- a/meta/64c97cb8.csv +++ b/meta/64c97cb8.csv @@ -100,7 +100,7 @@ Id,FileID,Domain,RepoName,FilePath,LineStart,LineEnd,GroundTruth,WithWords,Value 97076,e57a6dac,GitHub,64c97cb8,data/64c97cb8/test/e57a6dac.cs,71,71,T,F,28,37,F,F,,,,,2.73,9,F,F,F,Password 97077,e57a6dac,GitHub,64c97cb8,data/64c97cb8/test/e57a6dac.cs,37,37,T,F,28,37,F,F,,,,,2.37,9,F,F,F,Password 97078,f8402747,GitHub,64c97cb8,data/64c97cb8/test/f8402747.cs,30,30,T,F,34,47,F,F,,,,,3.11,13,F,F,F,Password -109489,37697095,GitHub,64c97cb8,data/64c97cb8/src/37697095.config,4,4,T,F,145,161,F,F,,,,,3.625,-1,F,F,F,Token +109489,37697095,GitHub,64c97cb8,data/64c97cb8/src/37697095.config,4,4,T,F,145,161,F,F,,,,,3.625,-1,F,F,F,Token:Key 109490,a73347eb,GitHub,64c97cb8,data/64c97cb8/src/a73347eb.cs,40,40,F,F,48,56,F,F,,,,,2.9219282,-1,F,F,F,Password 1113138,79ebb761,GitHub,64c97cb8,data/64c97cb8/src/79ebb761.cs,462,462,F,F,45,57,F,F,,,,,0.0,0,F,F,F,Credential 1113142,f8402747,GitHub,64c97cb8,data/64c97cb8/test/f8402747.cs,16,16,T,F,34,47,F,F,,,,,0.0,0,F,F,F,Password diff --git a/meta/69d49010.csv b/meta/69d49010.csv index 7e781966..368c7489 100644 --- a/meta/69d49010.csv +++ b/meta/69d49010.csv @@ -29,7 +29,7 @@ Id,FileID,Domain,RepoName,FilePath,LineStart,LineEnd,GroundTruth,WithWords,Value 16571,e7bdc301,GitHub,69d49010,data/69d49010/test/e7bdc301.py,71,71,T,F,34,48,F,F,,,,,0.0,0,F,F,F,Token 18645,0096a9db,GitHub,69d49010,data/69d49010/test/0096a9db,1,38,T,T,,,F,F,Any,Private,,Unknown,2.45,30,F,F,F,PEM Private Key 18914,d8bdaaee,GitHub,69d49010,data/69d49010/src/d8bdaaee.py,195,195,F,F,-1,-1,F,F,,,,,0.0,0,F,F,F,Key -24644,e7bdc301,GitHub,69d49010,data/69d49010/test/e7bdc301.py,20,20,F,F,18,43,F,F,,,,,0.0,0,F,F,F,Gitlab Incoming Email Token +24644,e7bdc301,GitHub,69d49010,data/69d49010/test/e7bdc301.py,20,20,F,F,18,43,F,F,,,,,0.0,0,F,F,F,Gitlab Incoming Email Token:Secret 26305,d6cfb4be,GitHub,69d49010,data/69d49010/src/d6cfb4be.py,67,67,F,F,,,F,F,,,,,0,0,F,F,F,API 27632,d9f42321,GitHub,69d49010,data/69d49010/test/d9f42321.py,75,75,T,F,61,68,F,F,CharsOnly,,,Secret,2.81,7,F,F,F,Secret 28928,1b7e760f,GitHub,69d49010,data/69d49010/test/1b7e760f,14,14,Template,F,45,51,F,F,Any,,,Unknown,2.12,7,F,F,F,Password:CMD Password @@ -44,7 +44,7 @@ Id,FileID,Domain,RepoName,FilePath,LineStart,LineEnd,GroundTruth,WithWords,Value 30628,53be66ac,GitHub,69d49010,data/69d49010/src/53be66ac.py,373,373,Template,F,32,35,F,F,CharsOnly,,,Secret,0.0,3,F,F,F,Password 31073,2b70d37a,GitHub,69d49010,data/69d49010/test/2b70d37a.py,72,72,Template,T,33,40,F,F,CharsOnly,,,Secret,2.52,7,F,F,F,Password 31087,3a17ec46,GitHub,69d49010,data/69d49010/test/3a17ec46.py,250,250,Template,T,25,38,F,F,Any,,,Secret,2.9,13,F,F,F,Password -31338,e7bdc301,GitHub,69d49010,data/69d49010/test/e7bdc301.py,21,21,Template,T,20,32,F,F,Any,,,Secret,2.56,12,F,F,F,Secret +31338,e7bdc301,GitHub,69d49010,data/69d49010/test/e7bdc301.py,21,21,F,T,20,32,F,F,Any,,,Secret,2.56,12,F,F,F,Secret 32506,7f3ac135,GitHub,69d49010,data/69d49010/test/7f3ac135.py,10,10,T,T,18,26,T,F,CharsOnly,,,Secret,2.75,8,F,F,F,URL Credentials 32566,85d52436,GitHub,69d49010,data/69d49010/test/85d52436.py,241,241,T,F,42,50,F,F,,,,,0.0,0,F,F,F,Password 32761,5aad918a,GitHub,69d49010,data/69d49010/test/5aad918a.py,250,250,T,F,35,41,F,F,CharsOnly,,,Unknown,2.25,6,F,F,F,Password diff --git a/meta/7c7558cf.csv b/meta/7c7558cf.csv index 4a0f13ee..55c3b30c 100644 --- a/meta/7c7558cf.csv +++ b/meta/7c7558cf.csv @@ -1,14 +1,14 @@ Id,FileID,Domain,RepoName,FilePath,LineStart,LineEnd,GroundTruth,WithWords,ValueStart,ValueEnd,InURL,InRuntimeParameter,CharacterSet,CryptographyKey,PredefinedPattern,VariableNameType,Entropy,Length,Base64Encode,HexEncode,URLEncode,Category 5,1ea02434,Github,7c7558cf,data/7c7558cf/other/1ea02434,40,40,T,F,46,50,T,F,,,,,,4,F,F,F,Auth:URL Credentials 15,024426f1,Github,7c7558cf,data/7c7558cf/src/024426f1.yml,20,20,T,F,31,39,F,F,,,,,0.0,8,F,F,F,Password -16,024426f1,Github,7c7558cf,data/7c7558cf/src/024426f1.yml,52,52,T,F,31,51,F,F,,,T,,0.0,20,F,F,F,AWS Client ID +16,024426f1,Github,7c7558cf,data/7c7558cf/src/024426f1.yml,52,52,T,F,31,51,F,F,,,T,,0.0,20,F,F,F,AWS Client ID:Key 21,024426f1,Github,7c7558cf,data/7c7558cf/src/024426f1.yml,53,53,T,F,35,75,F,F,,,,,0.0,40,F,F,F,Secret:Key 22,024426f1,Github,7c7558cf,data/7c7558cf/src/024426f1.yml,54,54,T,F,55,63,T,F,,,,,0.0,8,F,F,F,Auth:URL Credentials -126,c1f51cea,Github,7c7558cf,data/7c7558cf/src/c1f51cea.yml,41,41,T,F,31,51,F,F,,,T,,0.0,20,F,F,F,AWS Client ID +126,c1f51cea,Github,7c7558cf,data/7c7558cf/src/c1f51cea.yml,41,41,T,F,31,51,F,F,,,T,,0.0,20,F,F,F,AWS Client ID:Key 129,c1f51cea,Github,7c7558cf,data/7c7558cf/src/c1f51cea.yml,42,42,T,F,35,75,F,F,,,,,0.0,40,F,F,F,Key:Secret 130,c65b9371,Github,7c7558cf,data/7c7558cf/src/c65b9371.py,259,259,T,F,18,38,F,F,,,,,0.0,20,F,F,F,Password:Key:Secret 131,c7e23c27,Github,7c7558cf,data/7c7558cf/src/c7e23c27.yml,24,24,T,F,29,37,F,F,,,,,0.0,8,F,F,F,Password -132,c7e23c27,Github,7c7558cf,data/7c7558cf/src/c7e23c27.yml,59,59,T,F,31,51,F,F,,,T,,0.0,20,F,F,F,AWS Client ID +132,c7e23c27,Github,7c7558cf,data/7c7558cf/src/c7e23c27.yml,59,59,T,F,31,51,F,F,,,T,,0.0,20,F,F,F,AWS Client ID:Key 145,c7e23c27,Github,7c7558cf,data/7c7558cf/src/c7e23c27.yml,60,60,T,F,35,75,F,F,,,,,0.0,40,F,F,F,Secret:Key 146,c7e23c27,Github,7c7558cf,data/7c7558cf/src/c7e23c27.yml,61,61,T,F,54,62,F,F,,,,,0.0,8,F,F,F,Auth:URL Credentials 152,d6cd0538,Github,7c7558cf,data/7c7558cf/src/d6cd0538.py,30,30,F,F,34,72,F,F,,,,,,38,F,F,F,Password diff --git a/meta/a0cd6261.csv b/meta/a0cd6261.csv index 8059c132..37b58101 100644 --- a/meta/a0cd6261.csv +++ b/meta/a0cd6261.csv @@ -302,3 +302,5 @@ Id,FileID,Domain,RepoName,FilePath,LineStart,LineEnd,GroundTruth,WithWords,Value 1479431,7269210a,GitHub,a0cd6261,data/a0cd6261/test/7269210a.md,313,313,T,F,52,62,F,F,,,,,0.0,0,F,F,F,CMD Password 1479432,7269210a,GitHub,a0cd6261,data/a0cd6261/test/7269210a.md,314,314,T,F,49,61,F,F,,,,,0.0,0,F,F,F,CMD Password 1479563,7269210a,GitHub,a0cd6261,data/a0cd6261/test/7269210a.md,108,108,T,F,138,154,F,F,,,,,0.0,0,F,F,F,CMD Password +1479591,40bae391,GitHub,a0cd6261,data/a0cd6261/src/40bae391.txt,598,598,F,F,31,74,F,F,,,,,0.0,0,F,F,F,Key +1479592,29bb2e65,GitHub,a0cd6261,data/a0cd6261/test/29bb2e65.md,88,88,T,F,43,62,F,F,,,,,0.0,0,F,F,F,AWS Client ID:Key From 1403f13df17ce5134c780e42bdc0e84f8475a68b Mon Sep 17 00:00:00 2001 From: Roman Babenko Date: Wed, 4 Sep 2024 17:32:12 +0300 Subject: [PATCH 07/49] lostone --- .ci/benchmark.txt | 12 ++++++------ meta/48fd3902.csv | 1 + 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/.ci/benchmark.txt b/.ci/benchmark.txt index 67f3578b..35d740f5 100644 --- a/.ci/benchmark.txt +++ b/.ci/benchmark.txt @@ -1,6 +1,6 @@ -META MD5 8288d8a5b3f7edb9ff8f66a75b1fd6db +META MD5 2f44468089f8334e26213bf55a6b191b DATA MD5 8503780bbece7b6a6f1ba38f59897fdb -DATA: 16345596 interested lines. MARKUP: 62805 items +DATA: 16345596 interested lines. MARKUP: 62806 items FileType FileNumber ValidLines Positives Negatives Templates --------------- ------------ ------------ ----------- ----------- ----------- 194 28318 70 417 91 @@ -92,7 +92,7 @@ FileType FileNumber ValidLines Positives Negatives Templat .jwt 1 1 2 .key 83 2737 70 14 .ks 1 25 1 -.kt 123 20774 64 382 3 +.kt 123 20774 64 383 3 .l 1 982 1 .las 1 6656 36 .lasso 1 230 7 @@ -223,7 +223,7 @@ FileType FileNumber ValidLines Positives Negatives Templat .yml 419 36169 552 890 379 .zsh 6 872 12 .zsh-theme 1 97 1 -TOTAL: 10264 16345596 12189 50507 5102 +TOTAL: 10264 16345596 12189 50508 5102 credsweeper result_cnt : 0, lost_cnt : 0, true_cnt : 0, false_cnt : 0 Rules Positives Negatives Templates Reported TP FP TN FN FPR FNR ACC PRC RCL F1 ------------------------------ ----------- ----------- ----------- ---------- ---- ---- ----- ----- -------- -------- -------- ----- -------- ---- @@ -258,7 +258,7 @@ Grafana Provisioned API Key 22 1 0 JSON Web Token 170 61 0 0 0 61 170 0.000000 1.000000 0.264069 0.000000 Jira / Confluence PAT token 0 4 0 0 0 4 0 0.000000 1.000000 Jira 2FA 15 6 1 0 0 7 15 0.000000 1.000000 0.318182 0.000000 -Key 3905 15719 482 0 0 16201 3905 0.000000 1.000000 0.805779 0.000000 +Key 3905 15720 482 0 0 16202 3905 0.000000 1.000000 0.805789 0.000000 Nonce 91 49 0 0 0 49 91 0.000000 1.000000 0.350000 0.000000 Other 0 8291 1 0 0 8292 0 0.000000 1.000000 PEM Private Key 1019 1483 0 0 0 1483 1019 0.000000 1.000000 0.592726 0.000000 @@ -271,4 +271,4 @@ Token 648 4174 438 Twilio API Key 0 5 2 0 0 7 0 0.000000 1.000000 URL Credentials 210 151 220 0 0 371 210 0.000000 1.000000 0.638554 0.000000 UUID 1069 265 0 0 0 265 1069 0.000000 1.000000 0.198651 0.000000 - 12189 50507 5102 0 0 0 50507 12189 0.000000 1.000000 0.805586 0.000000 + 12189 50508 5102 0 0 0 50508 12189 0.000000 1.000000 0.805589 0.000000 diff --git a/meta/48fd3902.csv b/meta/48fd3902.csv index a4dfafd7..4b1f8acf 100644 --- a/meta/48fd3902.csv +++ b/meta/48fd3902.csv @@ -251,3 +251,4 @@ Id,FileID,Domain,RepoName,FilePath,LineStart,LineEnd,GroundTruth,WithWords,Value 1339604,c709b566,GitHub,48fd3902,data/48fd3902/test/c709b566.kt,96,96,T,F,48,84,F,F,,,,,0.0,0,F,F,F,UUID 1339605,c709b566,GitHub,48fd3902,data/48fd3902/test/c709b566.kt,131,131,T,F,43,79,F,F,,,,,0.0,0,F,F,F,UUID 1339606,c709b566,GitHub,48fd3902,data/48fd3902/test/c709b566.kt,135,135,T,F,44,80,F,F,,,,,0.0,0,F,F,F,UUID +1479593,4799bb8c,GitHub,48fd3902,data/48fd3902/test/4799bb8c.kt,565,565,F,F,46,89,F,F,,,,,0.0,0,F,F,F,Key From d50092f1109d74917c2303305476313ae934cb6f Mon Sep 17 00:00:00 2001 From: Roman Babenko Date: Wed, 4 Sep 2024 19:33:01 +0300 Subject: [PATCH 08/49] someF --- meta/99f9a6ea.csv | 2 ++ meta/a09d9e50.csv | 1 + 2 files changed, 3 insertions(+) diff --git a/meta/99f9a6ea.csv b/meta/99f9a6ea.csv index 4a99700d..0a48419a 100644 --- a/meta/99f9a6ea.csv +++ b/meta/99f9a6ea.csv @@ -97,3 +97,5 @@ Id,FileID,Domain,RepoName,FilePath,LineStart,LineEnd,GroundTruth,WithWords,Value 1138198,06572a96,GitHub,99f9a6ea,data/99f9a6ea/other/06572a96.md,3,3,F,F,,,F,F,,,,,0.0,0,F,F,F,Auth 1138199,06572a96,GitHub,99f9a6ea,data/99f9a6ea/other/06572a96.md,28,28,F,F,,,F,F,,,,,0.0,0,F,F,F,Auth 1341511,48fc739c,GitHub,99f9a6ea,data/99f9a6ea/other/48fc739c.md,78,78,F,F,89,126,F,F,,,,,0.0,0,F,F,F,API +1479594,154a978d,GitHub,99f9a6ea,data/99f9a6ea/test/154a978d.rb,426,426,F,F,64,71,F,F,,,,,0.0,0,F,F,F,Auth +1479595,154a978d,GitHub,99f9a6ea,data/99f9a6ea/test/154a978d.rb,445,445,F,F,64,71,F,F,,,,,0.0,0,F,F,F,Auth diff --git a/meta/a09d9e50.csv b/meta/a09d9e50.csv index 4819e1ab..e95e4308 100644 --- a/meta/a09d9e50.csv +++ b/meta/a09d9e50.csv @@ -407,3 +407,4 @@ Id,FileID,Domain,RepoName,FilePath,LineStart,LineEnd,GroundTruth,WithWords,Value 1479371,054074fe,GitHub,a09d9e50,data/a09d9e50/test/054074fe.ndjson,1675,1675,F,F,,,F,F,,,,,0.0,0,F,F,F,API 1479407,054074fe,GitHub,a09d9e50,data/a09d9e50/test/054074fe.ndjson,1187,1187,F,F,1122,1126,F,F,,,,,0.0,0,F,F,F,CMD Password 1479408,054074fe,GitHub,a09d9e50,data/a09d9e50/test/054074fe.ndjson,1404,1404,T,F,431,441,F,F,,,,,0.0,0,F,F,F,CMD ConvertTo-SecureString +1479596,054074fe,GitHub,a09d9e50,data/a09d9e50/test/054074fe.ndjson,367,367,F,F,1851,1867,F,F,,,,,0.0,0,F,F,F,Key:Token From c80ab4731cd40fa2758ae431fb6449baf0ffd0fe Mon Sep 17 00:00:00 2001 From: Roman Babenko Date: Wed, 4 Sep 2024 19:33:24 +0300 Subject: [PATCH 09/49] ciBM --- .ci/benchmark.txt | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/.ci/benchmark.txt b/.ci/benchmark.txt index 35d740f5..f5324573 100644 --- a/.ci/benchmark.txt +++ b/.ci/benchmark.txt @@ -1,6 +1,6 @@ -META MD5 2f44468089f8334e26213bf55a6b191b +META MD5 a3b7bc00de89d5a57a9df4d9d1e206e0 DATA MD5 8503780bbece7b6a6f1ba38f59897fdb -DATA: 16345596 interested lines. MARKUP: 62806 items +DATA: 16345596 interested lines. MARKUP: 62809 items FileType FileNumber ValidLines Positives Negatives Templates --------------- ------------ ------------ ----------- ----------- ----------- 194 28318 70 417 91 @@ -125,7 +125,7 @@ FileType FileNumber ValidLines Positives Negatives Templat .mqh 1 1023 2 .msg 1 26644 1 1 .mysql 1 36 2 -.ndjson 2 5006 74 239 2 +.ndjson 2 5006 74 241 2 .nix 4 211 12 .nolint 1 2 1 .odd 1 1281 43 @@ -159,7 +159,7 @@ FileType FileNumber ValidLines Positives Negatives Templat .pyx 2 1094 23 .r 4 62 6 3 1 .rake 2 51 2 -.rb 860 131838 259 3335 613 +.rb 860 131838 259 3337 613 .re 1 31 1 .red 1 159 1 .release 1 13 4 @@ -223,7 +223,7 @@ FileType FileNumber ValidLines Positives Negatives Templat .yml 419 36169 552 890 379 .zsh 6 872 12 .zsh-theme 1 97 1 -TOTAL: 10264 16345596 12189 50508 5102 +TOTAL: 10264 16345596 12189 50512 5102 credsweeper result_cnt : 0, lost_cnt : 0, true_cnt : 0, false_cnt : 0 Rules Positives Negatives Templates Reported TP FP TN FN FPR FNR ACC PRC RCL F1 ------------------------------ ----------- ----------- ----------- ---------- ---- ---- ----- ----- -------- -------- -------- ----- -------- ---- @@ -232,7 +232,7 @@ AWS Client ID 168 21 0 AWS Multi 75 16 0 0 0 16 75 0.000000 1.000000 0.175824 0.000000 AWS S3 Bucket 66 24 0 0 0 24 66 0.000000 1.000000 0.266667 0.000000 Atlassian Old PAT token 27 308 3 0 0 311 27 0.000000 1.000000 0.920118 0.000000 -Auth 419 2735 76 0 0 2811 419 0.000000 1.000000 0.870279 0.000000 +Auth 419 2737 76 0 0 2813 419 0.000000 1.000000 0.870359 0.000000 Azure Access Token 19 0 0 0 0 0 19 1.000000 0.000000 0.000000 BASE64 Private Key 7 4 0 0 0 4 7 0.000000 1.000000 0.363636 0.000000 BASE64 encoded PEM Private Key 7 0 0 0 0 0 7 1.000000 0.000000 0.000000 @@ -258,7 +258,7 @@ Grafana Provisioned API Key 22 1 0 JSON Web Token 170 61 0 0 0 61 170 0.000000 1.000000 0.264069 0.000000 Jira / Confluence PAT token 0 4 0 0 0 4 0 0.000000 1.000000 Jira 2FA 15 6 1 0 0 7 15 0.000000 1.000000 0.318182 0.000000 -Key 3905 15720 482 0 0 16202 3905 0.000000 1.000000 0.805789 0.000000 +Key 3905 15721 482 0 0 16203 3905 0.000000 1.000000 0.805799 0.000000 Nonce 91 49 0 0 0 49 91 0.000000 1.000000 0.350000 0.000000 Other 0 8291 1 0 0 8292 0 0.000000 1.000000 PEM Private Key 1019 1483 0 0 0 1483 1019 0.000000 1.000000 0.592726 0.000000 @@ -267,8 +267,8 @@ Salt 45 76 2 Secret 1298 1576 798 0 0 2374 1298 0.000000 1.000000 0.646514 0.000000 Seed 1 6 0 0 0 6 1 0.000000 1.000000 0.857143 0.000000 Slack Token 4 1 0 0 0 1 4 0.000000 1.000000 0.200000 0.000000 -Token 648 4174 438 0 0 4612 648 0.000000 1.000000 0.876806 0.000000 +Token 648 4175 438 0 0 4613 648 0.000000 1.000000 0.876830 0.000000 Twilio API Key 0 5 2 0 0 7 0 0.000000 1.000000 URL Credentials 210 151 220 0 0 371 210 0.000000 1.000000 0.638554 0.000000 UUID 1069 265 0 0 0 265 1069 0.000000 1.000000 0.198651 0.000000 - 12189 50508 5102 0 0 0 50508 12189 0.000000 1.000000 0.805589 0.000000 + 12189 50512 5102 0 0 0 50512 12189 0.000000 1.000000 0.805601 0.000000 From ac5b5e6ba669f6e02c774f42401b3a79f539afb3 Mon Sep 17 00:00:00 2001 From: Roman Babenko Date: Thu, 5 Sep 2024 09:23:47 +0300 Subject: [PATCH 10/49] lost_T --- .ci/benchmark.txt | 14 +++++++------- meta/850c2319.csv | 2 +- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/.ci/benchmark.txt b/.ci/benchmark.txt index f5324573..c393c8ed 100644 --- a/.ci/benchmark.txt +++ b/.ci/benchmark.txt @@ -1,6 +1,6 @@ -META MD5 a3b7bc00de89d5a57a9df4d9d1e206e0 -DATA MD5 8503780bbece7b6a6f1ba38f59897fdb -DATA: 16345596 interested lines. MARKUP: 62809 items +META MD5 95597baaefa3d69cdd0a1c631f9671af +DATA MD5 ed0b87ae9870d992d6f59dbbe1c6d520 +DATA: 16345596 interested lines. MARKUP: 62810 items FileType FileNumber ValidLines Positives Negatives Templates --------------- ------------ ------------ ----------- ----------- ----------- 194 28318 70 417 91 @@ -113,7 +113,7 @@ FileType FileNumber ValidLines Positives Negatives Templat .markdown 3 139 3 1 .markerb 3 12 3 .marko 1 21 2 -.md 675 149568 734 2392 623 +.md 675 149568 735 2392 623 .mdx 3 549 7 .mjml 1 18 1 .mjs 22 4424 75 340 @@ -223,7 +223,7 @@ FileType FileNumber ValidLines Positives Negatives Templat .yml 419 36169 552 890 379 .zsh 6 872 12 .zsh-theme 1 97 1 -TOTAL: 10264 16345596 12189 50512 5102 +TOTAL: 10264 16345596 12190 50512 5102 credsweeper result_cnt : 0, lost_cnt : 0, true_cnt : 0, false_cnt : 0 Rules Positives Negatives Templates Reported TP FP TN FN FPR FNR ACC PRC RCL F1 ------------------------------ ----------- ----------- ----------- ---------- ---- ---- ----- ----- -------- -------- -------- ----- -------- ---- @@ -262,7 +262,7 @@ Key 3905 15721 482 Nonce 91 49 0 0 0 49 91 0.000000 1.000000 0.350000 0.000000 Other 0 8291 1 0 0 8292 0 0.000000 1.000000 PEM Private Key 1019 1483 0 0 0 1483 1019 0.000000 1.000000 0.592726 0.000000 -Password 1848 7537 2702 0 0 10239 1848 0.000000 1.000000 0.847108 0.000000 +Password 1849 7537 2702 0 0 10239 1849 0.000000 1.000000 0.847038 0.000000 Salt 45 76 2 0 0 78 45 0.000000 1.000000 0.634146 0.000000 Secret 1298 1576 798 0 0 2374 1298 0.000000 1.000000 0.646514 0.000000 Seed 1 6 0 0 0 6 1 0.000000 1.000000 0.857143 0.000000 @@ -271,4 +271,4 @@ Token 648 4175 438 Twilio API Key 0 5 2 0 0 7 0 0.000000 1.000000 URL Credentials 210 151 220 0 0 371 210 0.000000 1.000000 0.638554 0.000000 UUID 1069 265 0 0 0 265 1069 0.000000 1.000000 0.198651 0.000000 - 12189 50512 5102 0 0 0 50512 12189 0.000000 1.000000 0.805601 0.000000 + 12190 50512 5102 0 0 0 50512 12190 0.000000 1.000000 0.805588 0.000000 diff --git a/meta/850c2319.csv b/meta/850c2319.csv index 5363818b..2073690e 100644 --- a/meta/850c2319.csv +++ b/meta/850c2319.csv @@ -110,4 +110,4 @@ Id,FileID,Domain,RepoName,FilePath,LineStart,LineEnd,GroundTruth,WithWords,Value 130484,128a3f9f,GitHub,850c2319,data/850c2319/other/128a3f9f.md,15,15,T,F,12,22,F,F,Any,,,Secret,1.0,2,F,F,F,Password 133804,a746be65,GitHub,850c2319,data/850c2319/src/a746be65.go,642,642,F,F,-1,-1,F,F,,,,,0.0,0,F,F,F,Key 133805,a746be65,GitHub,850c2319,data/850c2319/src/a746be65.go,643,643,F,F,-1,-1,F,F,,,,,0.0,0,F,F,F,Key - +1479603,5ac4f856,GitHub,850c2319,data/850c2319/other/5ac4f856.md,28,28,T,F,15,25,F,F,,,,,0.0,0,F,F,F,Password From cfa6f331b4020faee42bae8bb77247eaeaad3e08 Mon Sep 17 00:00:00 2001 From: Roman Babenko Date: Thu, 5 Sep 2024 09:53:25 +0300 Subject: [PATCH 11/49] short and color --- meta/387016a6.csv | 2 +- meta/6c73b80a.csv | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/meta/387016a6.csv b/meta/387016a6.csv index b3aa012a..c2f022fc 100644 --- a/meta/387016a6.csv +++ b/meta/387016a6.csv @@ -1112,4 +1112,4 @@ Id,FileID,Domain,RepoName,FilePath,LineStart,LineEnd,GroundTruth,WithWords,Value 1027483,89ce63c9,GitHub,387016a6,data/387016a6/test/89ce63c9.yml,320,320,T,T,39,71,F,F,Any,,,Secret,3.2,15,F,F,F,Auth 1027484,89ce63c9,GitHub,387016a6,data/387016a6/test/89ce63c9.yml,336,336,T,T,39,71,F,F,Any,,,Secret,3.2,15,F,F,F,Auth 1027485,89ce63c9,GitHub,387016a6,data/387016a6/test/89ce63c9.yml,348,348,T,T,39,71,F,F,Any,,,Secret,3.2,15,F,F,F,Auth -1238210,499a20a4,GitHub,387016a6,data/387016a6/src/499a20a4.asciidoc,187,187,T,F,81,90,F,F,,,,,0.0,0,F,F,F,Token +1238210,499a20a4,GitHub,387016a6,data/387016a6/src/499a20a4.asciidoc,187,187,F,F,,,F,F,,,,,0.0,0,F,F,F,Token diff --git a/meta/6c73b80a.csv b/meta/6c73b80a.csv index ae37b12d..fcf1c1af 100644 --- a/meta/6c73b80a.csv +++ b/meta/6c73b80a.csv @@ -329,7 +329,7 @@ Id,FileID,Domain,RepoName,FilePath,LineStart,LineEnd,GroundTruth,WithWords,Value 35347,88763956,GitHub,6c73b80a,data/6c73b80a/test/88763956.go,3524,3524,T,T,32,36,F,F,Any,,,Secret,1.46,5,F,F,F,Password 35360,515f4ca6,GitHub,6c73b80a,data/6c73b80a/test/515f4ca6.go,4286,4286,Template,F,26,29,F,F,Any,,,Secret,1.58,3,F,F,F,Password 35371,f1bada03,GitHub,6c73b80a,data/6c73b80a/test/f1bada03.go,3600,3600,Template,F,29,32,F,F,Any,,,Secret,1.58,3,F,F,F,Password -35381,515f4ca6,GitHub,6c73b80a,data/6c73b80a/test/515f4ca6.go,2570,2570,T,F,26,30,F,F,Any,,,Secret,1.5,4,F,F,F,Password +35381,515f4ca6,GitHub,6c73b80a,data/6c73b80a/test/515f4ca6.go,2570,2570,F,F,26,29,F,F,Any,,,Short,1.5,4,F,F,F,Password 35382,633f9ac2,GitHub,6c73b80a,data/6c73b80a/test/633f9ac2.go,3515,3515,Template,F,36,39,F,F,Any,,,Secret,1.58,3,F,F,F,Password 35383,88341713,GitHub,6c73b80a,data/6c73b80a/test/88341713.go,3183,3183,Template,F,25,28,F,F,Any,,,Secret,1.58,3,F,F,F,Password 35390,515f4ca6,GitHub,6c73b80a,data/6c73b80a/test/515f4ca6.go,2556,2556,T,T,28,33,F,F,Any,,,Secret,1.82,6,F,F,F,Password @@ -428,7 +428,7 @@ Id,FileID,Domain,RepoName,FilePath,LineStart,LineEnd,GroundTruth,WithWords,Value 37426,88763956,GitHub,6c73b80a,data/6c73b80a/test/88763956.go,3561,3561,T,T,32,36,F,F,Any,,,Secret,1.46,5,F,F,F,Password 37429,515f4ca6,GitHub,6c73b80a,data/6c73b80a/test/515f4ca6.go,4412,4412,Template,F,26,29,F,F,Any,,,Secret,1.58,3,F,F,F,Password 37430,515f4ca6,GitHub,6c73b80a,data/6c73b80a/test/515f4ca6.go,4434,4434,Template,F,26,29,F,F,Any,,,Secret,1.58,3,F,F,F,Password -37439,515f4ca6,GitHub,6c73b80a,data/6c73b80a/test/515f4ca6.go,2666,2666,T,F,26,30,F,F,Any,,,Secret,1.5,4,F,F,F,Password +37439,515f4ca6,GitHub,6c73b80a,data/6c73b80a/test/515f4ca6.go,2666,2666,F,F,26,30,F,F,Any,,,Short,1.5,4,F,F,F,Password 37440,515f4ca6,GitHub,6c73b80a,data/6c73b80a/test/515f4ca6.go,2660,2660,T,T,28,33,F,F,Any,,,Secret,1.82,6,F,F,F,Password 37441,633f9ac2,GitHub,6c73b80a,data/6c73b80a/test/633f9ac2.go,3673,3673,Template,F,37,40,F,F,Any,,,Secret,1.58,3,F,F,F,Password 37442,633f9ac2,GitHub,6c73b80a,data/6c73b80a/test/633f9ac2.go,3708,3708,Template,F,37,40,F,F,Any,,,Secret,1.58,3,F,F,F,Password From 04b851acbfaee62956067d77f196897422495d40 Mon Sep 17 00:00:00 2001 From: Roman Babenko Date: Thu, 5 Sep 2024 09:55:39 +0300 Subject: [PATCH 12/49] BM --- .ci/benchmark.txt | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.ci/benchmark.txt b/.ci/benchmark.txt index c393c8ed..767f408c 100644 --- a/.ci/benchmark.txt +++ b/.ci/benchmark.txt @@ -1,5 +1,5 @@ -META MD5 95597baaefa3d69cdd0a1c631f9671af -DATA MD5 ed0b87ae9870d992d6f59dbbe1c6d520 +META MD5 e0d23936b5acb88545e766e8e06e8f3f +DATA MD5 d1b647616f0a004042d24c64a77327a0 DATA: 16345596 interested lines. MARKUP: 62810 items FileType FileNumber ValidLines Positives Negatives Templates --------------- ------------ ------------ ----------- ----------- ----------- @@ -8,7 +8,7 @@ FileType FileNumber ValidLines Positives Negatives Templat .admx 1 26 1 .adoc 1 158 13 6 1 .api 2 118 4 -.asciidoc 96 14471 50 347 27 +.asciidoc 96 14471 49 348 27 .axaml 5 286 5 .backup 1 62 2 1 .bash 2 2158 2 1 @@ -63,7 +63,7 @@ FileType FileNumber ValidLines Positives Negatives Templat .gd 1 37 1 .gml 3 3075 16 .gni 3 5017 19 -.go 1080 566476 695 4122 738 +.go 1080 566476 693 4124 738 .golden 5 1168 1 13 29 .gradle 45 3265 4 90 100 .graphql 7 420 13 @@ -223,7 +223,7 @@ FileType FileNumber ValidLines Positives Negatives Templat .yml 419 36169 552 890 379 .zsh 6 872 12 .zsh-theme 1 97 1 -TOTAL: 10264 16345596 12190 50512 5102 +TOTAL: 10264 16345596 12187 50515 5102 credsweeper result_cnt : 0, lost_cnt : 0, true_cnt : 0, false_cnt : 0 Rules Positives Negatives Templates Reported TP FP TN FN FPR FNR ACC PRC RCL F1 ------------------------------ ----------- ----------- ----------- ---------- ---- ---- ----- ----- -------- -------- -------- ----- -------- ---- @@ -262,13 +262,13 @@ Key 3905 15721 482 Nonce 91 49 0 0 0 49 91 0.000000 1.000000 0.350000 0.000000 Other 0 8291 1 0 0 8292 0 0.000000 1.000000 PEM Private Key 1019 1483 0 0 0 1483 1019 0.000000 1.000000 0.592726 0.000000 -Password 1849 7537 2702 0 0 10239 1849 0.000000 1.000000 0.847038 0.000000 +Password 1847 7539 2702 0 0 10241 1847 0.000000 1.000000 0.847204 0.000000 Salt 45 76 2 0 0 78 45 0.000000 1.000000 0.634146 0.000000 Secret 1298 1576 798 0 0 2374 1298 0.000000 1.000000 0.646514 0.000000 Seed 1 6 0 0 0 6 1 0.000000 1.000000 0.857143 0.000000 Slack Token 4 1 0 0 0 1 4 0.000000 1.000000 0.200000 0.000000 -Token 648 4175 438 0 0 4613 648 0.000000 1.000000 0.876830 0.000000 +Token 647 4176 438 0 0 4614 647 0.000000 1.000000 0.877020 0.000000 Twilio API Key 0 5 2 0 0 7 0 0.000000 1.000000 URL Credentials 210 151 220 0 0 371 210 0.000000 1.000000 0.638554 0.000000 UUID 1069 265 0 0 0 265 1069 0.000000 1.000000 0.198651 0.000000 - 12190 50512 5102 0 0 0 50512 12190 0.000000 1.000000 0.805588 0.000000 + 12187 50515 5102 0 0 0 50515 12187 0.000000 1.000000 0.805636 0.000000 From 621925c09e76496449051e8a65548eac9cae194c Mon Sep 17 00:00:00 2001 From: Roman Babenko Date: Thu, 5 Sep 2024 12:51:15 +0300 Subject: [PATCH 13/49] [skip actions] [auxiliary] 2024-09-05T12:51:15+03:00 --- meta/850c2319.csv | 1 + 1 file changed, 1 insertion(+) diff --git a/meta/850c2319.csv b/meta/850c2319.csv index 2073690e..d83ca6f7 100644 --- a/meta/850c2319.csv +++ b/meta/850c2319.csv @@ -111,3 +111,4 @@ Id,FileID,Domain,RepoName,FilePath,LineStart,LineEnd,GroundTruth,WithWords,Value 133804,a746be65,GitHub,850c2319,data/850c2319/src/a746be65.go,642,642,F,F,-1,-1,F,F,,,,,0.0,0,F,F,F,Key 133805,a746be65,GitHub,850c2319,data/850c2319/src/a746be65.go,643,643,F,F,-1,-1,F,F,,,,,0.0,0,F,F,F,Key 1479603,5ac4f856,GitHub,850c2319,data/850c2319/other/5ac4f856.md,28,28,T,F,15,25,F,F,,,,,0.0,0,F,F,F,Password + From c6aec6a6a7bba86d3c4b93dffcf9d286d351d321 Mon Sep 17 00:00:00 2001 From: Roman Babenko Date: Thu, 5 Sep 2024 12:51:43 +0300 Subject: [PATCH 14/49] UpdMeta --- .ci/benchmark.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ci/benchmark.txt b/.ci/benchmark.txt index 767f408c..0098964b 100644 --- a/.ci/benchmark.txt +++ b/.ci/benchmark.txt @@ -1,4 +1,4 @@ -META MD5 e0d23936b5acb88545e766e8e06e8f3f +META MD5 9995854784989502b344d514aea0faed DATA MD5 d1b647616f0a004042d24c64a77327a0 DATA: 16345596 interested lines. MARKUP: 62810 items FileType FileNumber ValidLines Positives Negatives Templates From 413069ed64a186d9f9c720afd96136db829335aa Mon Sep 17 00:00:00 2001 From: Roman Babenko Date: Thu, 5 Sep 2024 16:24:51 +0300 Subject: [PATCH 15/49] Corrections --- .ci/benchmark.txt | 46 +++++++++++++++++++++++----------------------- meta/0436af4a.csv | 1 + meta/0f133e09.csv | 10 +++++----- meta/288eaba8.csv | 2 +- meta/2ba83c6a.csv | 2 +- meta/472d4c24.csv | 2 +- meta/48fd3902.csv | 4 ++-- meta/57c424f8.csv | 4 ++-- meta/680ef0c1.csv | 2 +- meta/90aebe4a.csv | 1 + meta/a15774b8.csv | 2 +- meta/b6b2487d.csv | 2 +- meta/bbb4193f.csv | 4 ++-- meta/c2d95db7.csv | 4 ++-- meta/f710ac3c.csv | 2 +- 15 files changed, 45 insertions(+), 43 deletions(-) diff --git a/.ci/benchmark.txt b/.ci/benchmark.txt index 0098964b..665ccc80 100644 --- a/.ci/benchmark.txt +++ b/.ci/benchmark.txt @@ -1,9 +1,9 @@ -META MD5 9995854784989502b344d514aea0faed -DATA MD5 d1b647616f0a004042d24c64a77327a0 -DATA: 16345596 interested lines. MARKUP: 62810 items +META MD5 f8ea5ee867e628f3add2f09d730f72df +DATA MD5 79ac8163a2132e6d6588718e81ced074 +DATA: 16345596 interested lines. MARKUP: 62812 items FileType FileNumber ValidLines Positives Negatives Templates --------------- ------------ ------------ ----------- ----------- ----------- - 194 28318 70 417 91 + 194 28318 71 417 90 .1 2 641 2 5 .admx 1 26 1 .adoc 1 158 13 6 1 @@ -35,7 +35,7 @@ FileType FileNumber ValidLines Positives Negatives Templat .creds 1 10 1 1 .crlf 1 27 1 .crt 2 4979 211 -.cs 268 79532 160 894 94 +.cs 268 79532 160 895 94 .cshtml 5 180 12 .csp 3 379 9 .csproj 1 14 1 @@ -55,7 +55,7 @@ FileType FileNumber ValidLines Positives Negatives Templat .erb 13 323 27 .erl 4 96 7 .ex 25 4968 5 98 5 -.example 17 1838 74 38 54 +.example 17 1838 75 38 53 .exs 24 4842 8 187 4 .ext 5 211 1 4 2 .fsproj 1 75 1 2 @@ -63,7 +63,7 @@ FileType FileNumber ValidLines Positives Negatives Templat .gd 1 37 1 .gml 3 3075 16 .gni 3 5017 19 -.go 1080 566476 693 4124 738 +.go 1080 566476 694 4123 738 .golden 5 1168 1 13 29 .gradle 45 3265 4 90 100 .graphql 7 420 13 @@ -82,11 +82,11 @@ FileType FileNumber ValidLines Positives Negatives Templat .ipynb 1 134 5 .j 1 241 4 .j2 30 5530 6 186 10 -.java 621 134132 363 1364 170 +.java 621 134132 361 1366 170 .jenkinsfile 1 58 2 6 .jinja2 1 64 2 -.js 659 536413 535 2496 328 -.json 851 13046493 1071 10911 140 +.js 659 536413 536 2496 328 +.json 851 13046493 1072 10910 140 .jsp 13 3202 1 40 .jsx 7 857 19 .jwt 1 1 2 @@ -136,7 +136,7 @@ FileType FileNumber ValidLines Positives Negatives Templat .pbxproj 1 941 2 .pem 48 1169 47 8 .php 371 75710 129 1620 79 -.pl 16 14727 6 34 +.pl 16 14727 7 33 .pm 3 744 7 .po 3 2994 15 .pod 9 1859 1 23 @@ -146,20 +146,20 @@ FileType FileNumber ValidLines Positives Negatives Templat .ppk 1 45 36 .private 1 15 1 .proj 1 85 5 -.properties 48 1621 54 27 32 +.properties 48 1621 54 29 32 .proto 5 5768 2 49 .ps1 16 8509 15 74 2 .ps1xml 1 5022 1 .pug 2 193 2 .purs 1 69 4 .pxd 1 150 5 2 -.py 890 291553 683 3301 723 +.py 890 291553 684 3300 723 .pyi 4 1361 9 .pyp 1 167 1 .pyx 2 1094 23 .r 4 62 6 3 1 .rake 2 51 2 -.rb 860 131838 259 3337 613 +.rb 860 131838 260 3337 612 .re 1 31 1 .red 1 159 1 .release 1 13 4 @@ -179,7 +179,7 @@ FileType FileNumber ValidLines Positives Negatives Templat .scala 40 5071 22 101 .scss 16 8553 32 1 .secrets 1 11 1 -.sh 143 21525 54 480 30 +.sh 143 21525 55 479 30 .slim 1 153 1 2 .smali 1 775 18 .snap 3 1708 9 30 2 @@ -209,7 +209,7 @@ FileType FileNumber ValidLines Positives Negatives Templat .ts 583 106730 159 1800 201 .tsx 54 7914 1 114 5 .ttar 1 452 1 -.txt 440 78102 5272 6373 49 +.txt 440 78102 5277 6368 49 .utf8 1 77 2 .vsixmanifest 1 36 1 .vsmdi 1 6 2 @@ -220,10 +220,10 @@ FileType FileNumber ValidLines Positives Negatives Templat .xml 9 689 9 .xsl 1 311 1 .yaml 137 19004 125 345 42 -.yml 419 36169 552 890 379 +.yml 419 36169 555 888 378 .zsh 6 872 12 .zsh-theme 1 97 1 -TOTAL: 10264 16345596 12187 50515 5102 +TOTAL: 10264 16345596 12202 50508 5098 credsweeper result_cnt : 0, lost_cnt : 0, true_cnt : 0, false_cnt : 0 Rules Positives Negatives Templates Reported TP FP TN FN FPR FNR ACC PRC RCL F1 ------------------------------ ----------- ----------- ----------- ---------- ---- ---- ----- ----- -------- -------- -------- ----- -------- ---- @@ -232,7 +232,7 @@ AWS Client ID 168 21 0 AWS Multi 75 16 0 0 0 16 75 0.000000 1.000000 0.175824 0.000000 AWS S3 Bucket 66 24 0 0 0 24 66 0.000000 1.000000 0.266667 0.000000 Atlassian Old PAT token 27 308 3 0 0 311 27 0.000000 1.000000 0.920118 0.000000 -Auth 419 2737 76 0 0 2813 419 0.000000 1.000000 0.870359 0.000000 +Auth 419 2738 76 0 0 2814 419 0.000000 1.000000 0.870399 0.000000 Azure Access Token 19 0 0 0 0 0 19 1.000000 0.000000 0.000000 BASE64 Private Key 7 4 0 0 0 4 7 0.000000 1.000000 0.363636 0.000000 BASE64 encoded PEM Private Key 7 0 0 0 0 0 7 1.000000 0.000000 0.000000 @@ -258,17 +258,17 @@ Grafana Provisioned API Key 22 1 0 JSON Web Token 170 61 0 0 0 61 170 0.000000 1.000000 0.264069 0.000000 Jira / Confluence PAT token 0 4 0 0 0 4 0 0.000000 1.000000 Jira 2FA 15 6 1 0 0 7 15 0.000000 1.000000 0.318182 0.000000 -Key 3905 15721 482 0 0 16203 3905 0.000000 1.000000 0.805799 0.000000 +Key 3908 15720 482 0 0 16202 3908 0.000000 1.000000 0.805669 0.000000 Nonce 91 49 0 0 0 49 91 0.000000 1.000000 0.350000 0.000000 Other 0 8291 1 0 0 8292 0 0.000000 1.000000 PEM Private Key 1019 1483 0 0 0 1483 1019 0.000000 1.000000 0.592726 0.000000 -Password 1847 7539 2702 0 0 10241 1847 0.000000 1.000000 0.847204 0.000000 +Password 1854 7537 2698 0 0 10235 1854 0.000000 1.000000 0.846637 0.000000 Salt 45 76 2 0 0 78 45 0.000000 1.000000 0.634146 0.000000 Secret 1298 1576 798 0 0 2374 1298 0.000000 1.000000 0.646514 0.000000 Seed 1 6 0 0 0 6 1 0.000000 1.000000 0.857143 0.000000 Slack Token 4 1 0 0 0 1 4 0.000000 1.000000 0.200000 0.000000 -Token 647 4176 438 0 0 4614 647 0.000000 1.000000 0.877020 0.000000 +Token 652 4171 438 0 0 4609 652 0.000000 1.000000 0.876069 0.000000 Twilio API Key 0 5 2 0 0 7 0 0.000000 1.000000 URL Credentials 210 151 220 0 0 371 210 0.000000 1.000000 0.638554 0.000000 UUID 1069 265 0 0 0 265 1069 0.000000 1.000000 0.198651 0.000000 - 12187 50515 5102 0 0 0 50515 12187 0.000000 1.000000 0.805636 0.000000 + 12202 50508 5098 0 0 0 50508 12202 0.000000 1.000000 0.805422 0.000000 diff --git a/meta/0436af4a.csv b/meta/0436af4a.csv index 065d552b..fae1ecf3 100644 --- a/meta/0436af4a.csv +++ b/meta/0436af4a.csv @@ -180,3 +180,4 @@ Id,FileID,Domain,RepoName,FilePath,LineStart,LineEnd,GroundTruth,WithWords,Value 1370690,697a0f01,GitHub,0436af4a,data/0436af4a/test/697a0f01.cs,66,66,T,F,31,67,F,F,,,,,0.0,0,F,F,F,UUID 1371381,6d6c3e78,GitHub,0436af4a,data/0436af4a/src/6d6c3e78.cs,151,151,F,F,51,61,F,F,,,,,0.0,0,F,F,F,Auth 1381055,8a1f6cd5,GitHub,0436af4a,data/0436af4a/test/8a1f6cd5.cs,269,269,F,F,,,F,F,,,,,0.0,0,F,F,F,Password +1479604,e4cc8527,GitHub,0436af4a,data/0436af4a/src/e4cc8527.cs,26,26,F,F,59,77,F,F,,,,,0.0,0,F,F,F,Auth diff --git a/meta/0f133e09.csv b/meta/0f133e09.csv index 285aa92a..ef0ae2b7 100644 --- a/meta/0f133e09.csv +++ b/meta/0f133e09.csv @@ -49,7 +49,7 @@ Id,FileID,Domain,RepoName,FilePath,LineStart,LineEnd,GroundTruth,WithWords,Value 5313,f2c0348c,GitHub,0f133e09,data/0f133e09/test/f2c0348c.txt,10,10,F,F,,,F,F,,,,,0,0,F,F,F,Token 5316,75bffd28,GitHub,0f133e09,data/0f133e09/test/75bffd28.txt,43,43,F,F,,,F,F,,,,,0,0,F,F,F,Auth 5318,75bffd28,GitHub,0f133e09,data/0f133e09/test/75bffd28.txt,32,32,F,F,,,F,F,,,,,0,0,F,F,F,Auth -5319,5ff4b6b7,GitHub,0f133e09,data/0f133e09/test/5ff4b6b7.txt,10,10,F,F,,,F,F,,,,,0,0,F,F,F,Token +5319,5ff4b6b7,GitHub,0f133e09,data/0f133e09/test/5ff4b6b7.txt,10,10,T,F,210,250,F,F,,,,,0,0,F,F,F,Token 5323,c92f896c,GitHub,0f133e09,data/0f133e09/test/c92f896c.txt,32,32,F,F,-1,-1,F,F,,,,,0.0,0,F,F,F,Other 5326,bb7be143,GitHub,0f133e09,data/0f133e09/test/bb7be143.txt,186,186,F,F,,,F,F,,,,,0,0,F,F,F,Bitbucket Client ID 5327,4573b88d,GitHub,0f133e09,data/0f133e09/test/4573b88d.txt,10,10,F,F,,,F,F,,,,,0,0,F,F,F,Bitbucket Client ID @@ -423,12 +423,12 @@ Id,FileID,Domain,RepoName,FilePath,LineStart,LineEnd,GroundTruth,WithWords,Value 10131,bb7be143,GitHub,0f133e09,data/0f133e09/test/bb7be143.txt,120,120,F,F,,,F,F,,,,,0,0,F,F,F,Auth 10132,f89519ab,GitHub,0f133e09,data/0f133e09/test/f89519ab.txt,10,10,F,F,,,F,F,,,,,0,0,F,F,F,Bitbucket Client ID 10135,c37b8d70,GitHub,0f133e09,data/0f133e09/test/c37b8d70.txt,10,10,F,F,,,F,F,,,,,0,0,F,F,F,Bitbucket Client ID -10136,5a7f3ab7,GitHub,0f133e09,data/0f133e09/test/5a7f3ab7.txt,10,10,F,F,,,F,F,,,,,0,0,F,F,F,Token +10136,5a7f3ab7,GitHub,0f133e09,data/0f133e09/test/5a7f3ab7.txt,10,10,T,F,226,266,F,F,,,,,0,0,F,F,F,Token 10137,983d2575,GitHub,0f133e09,data/0f133e09/test/983d2575.txt,10,10,F,F,,,F,F,,,,,0,0,F,F,F,Token 10138,0d46126a,GitHub,0f133e09,data/0f133e09/test/0d46126a.txt,10,10,F,F,,,F,F,,,,,0,0,F,F,F,Token -10139,5a7f3ab7,GitHub,0f133e09,data/0f133e09/test/5a7f3ab7.txt,43,43,F,F,,,F,F,,,,,0,0,F,F,F,Token -10140,5a7f3ab7,GitHub,0f133e09,data/0f133e09/test/5a7f3ab7.txt,21,21,F,F,,,F,F,,,,,0,0,F,F,F,Token -10141,5a7f3ab7,GitHub,0f133e09,data/0f133e09/test/5a7f3ab7.txt,32,32,F,F,,,F,F,,,,,0,0,F,F,F,Token +10139,5a7f3ab7,GitHub,0f133e09,data/0f133e09/test/5a7f3ab7.txt,43,43,T,F,232,272,F,F,,,,,0,0,F,F,F,Token +10140,5a7f3ab7,GitHub,0f133e09,data/0f133e09/test/5a7f3ab7.txt,21,21,T,F,232,272,F,F,,,,,0,0,F,F,F,Token +10141,5a7f3ab7,GitHub,0f133e09,data/0f133e09/test/5a7f3ab7.txt,32,32,T,F,223,263,F,F,,,,,0,0,F,F,F,Token 10142,4bc35222,GitHub,0f133e09,data/0f133e09/test/4bc35222.txt,10,10,F,F,,,F,F,,,,,0,0,F,F,F,Token 10143,6ec6580c,GitHub,0f133e09,data/0f133e09/test/6ec6580c.txt,10,10,F,F,,,F,F,,,,,0,0,F,F,F,Token 10149,59a5a023,GitHub,0f133e09,data/0f133e09/test/59a5a023.txt,21,21,F,F,,,F,F,,,,,0,0,F,F,F,Bitbucket Client ID diff --git a/meta/288eaba8.csv b/meta/288eaba8.csv index b53da45f..6a0c3e03 100644 --- a/meta/288eaba8.csv +++ b/meta/288eaba8.csv @@ -23,7 +23,7 @@ Id,FileID,Domain,RepoName,FilePath,LineStart,LineEnd,GroundTruth,WithWords,Value 29932,d2417b18,GitHub,288eaba8,data/288eaba8/test/d2417b18.js,11,11,T,T,17,38,F,F,Any,,,Secret,3.22,21,F,F,F,Password 30690,e370d734,GitHub,288eaba8,data/288eaba8/test/e370d734.js,141,141,T,T,24,37,F,F,Any,,,Secret,1.71,13,F,F,F,Password 30875,e370d734,GitHub,288eaba8,data/288eaba8/test/e370d734.js,149,149,T,F,64,74,F,F,,,,,0.0,0,F,F,F,Password -31159,0d52ab5c,GitHub,288eaba8,data/288eaba8/src/0d52ab5c.yml,74,74,Template,T,26,34,F,F,CharsOnly,,,Secret,2.5,8,F,F,F,Password +31159,0d52ab5c,GitHub,288eaba8,data/288eaba8/src/0d52ab5c.yml,74,74,T,T,26,34,F,F,CharsOnly,,,Secret,2.5,8,F,F,F,Password 31168,0d52ab5c,GitHub,288eaba8,data/288eaba8/src/0d52ab5c.yml,34,34,Template,T,28,44,F,F,CharsOnly,,,Secret,2.98,16,F,F,F,Password 31170,0d52ab5c,GitHub,288eaba8,data/288eaba8/src/0d52ab5c.yml,37,37,T,T,23,35,F,F,CharsOnly,,,Secret,3.02,12,F,F,F,Password 34337,a9d11eea,GitHub,288eaba8,data/288eaba8/test/a9d11eea.js,35,35,T,T,15,35,F,F,Any,,,Secret,3.68,20,F,F,F,Gitlab Feed Token:Password diff --git a/meta/2ba83c6a.csv b/meta/2ba83c6a.csv index 31468c6f..d863c743 100644 --- a/meta/2ba83c6a.csv +++ b/meta/2ba83c6a.csv @@ -978,7 +978,7 @@ Id,FileID,Domain,RepoName,FilePath,LineStart,LineEnd,GroundTruth,WithWords,Value 28081,3fc35e88,GitHub,2ba83c6a,data/2ba83c6a/test/3fc35e88.t,104,104,Template,F,33,41,F,F,Any,,,Unknown,2.62,8,F,F,F,Password 28126,4506daf1,GitHub,2ba83c6a,data/2ba83c6a/src/4506daf1.c,152,152,F,F,10,-1,F,F,,,,,0.0,0,F,F,F,Other 28127,f1b98a0f,GitHub,2ba83c6a,data/2ba83c6a/src/f1b98a0f.c,415,415,F,F,10,-1,F,F,,,,,0.0,0,F,F,F,Other -28351,afbc9d54,GitHub,2ba83c6a,data/2ba83c6a/src/afbc9d54.pl,113,113,F,F,,,F,F,,,,,0,0,F,F,F,Key +28351,afbc9d54,GitHub,2ba83c6a,data/2ba83c6a/src/afbc9d54.pl,113,113,T,F,7,23,F,F,,,,,0,0,F,F,F,Key 28455,07d8fbd6,GitHub,2ba83c6a,data/2ba83c6a/test/07d8fbd6.t,45,45,Template,T,77,83,F,F,Any,,,Secret,2.25,6,F,F,F,Password:Secret 28459,07d8fbd6,GitHub,2ba83c6a,data/2ba83c6a/test/07d8fbd6.t,48,48,Template,T,79,85,F,F,Any,,,Secret,2.25,6,F,F,F,Password:Secret 28460,07d8fbd6,GitHub,2ba83c6a,data/2ba83c6a/test/07d8fbd6.t,51,51,Template,T,74,80,F,F,Any,,,Secret,2.25,6,F,F,F,Key:Password diff --git a/meta/472d4c24.csv b/meta/472d4c24.csv index 790ccb01..915592dd 100644 --- a/meta/472d4c24.csv +++ b/meta/472d4c24.csv @@ -64,7 +64,7 @@ Id,FileID,Domain,RepoName,FilePath,LineStart,LineEnd,GroundTruth,WithWords,Value 34786,d17da2d4,GitHub,472d4c24,data/472d4c24/test/d17da2d4.rb,100,100,Template,T,59,63,F,F,CharsOnly,,,Secret,1.5,4,F,F,F,Password 34787,7b88edb5,GitHub,472d4c24,data/472d4c24/test/7b88edb5.rb,30,30,T,T,66,74,F,F,CharsOnly,,,Secret,3.0,8,F,F,F,Password 36080,4beb1f7a,GitHub,472d4c24,data/472d4c24/test/4beb1f7a.rb,146,146,Template,T,36,45,F,F,CharsOnly,,,Secret,2.73,9,F,F,F,Password -36081,4beb1f7a,GitHub,472d4c24,data/472d4c24/test/4beb1f7a.rb,145,145,Template,T,23,32,F,F,CharsOnly,,,Secret,2.73,9,F,F,F,Password +36081,4beb1f7a,GitHub,472d4c24,data/472d4c24/test/4beb1f7a.rb,145,145,T,T,23,32,F,F,CharsOnly,,,Secret,2.73,9,F,F,F,Password 36659,5ff8e937,GitHub,472d4c24,data/472d4c24/test/5ff8e937.rb,129,129,Template,T,22,30,F,F,Any,,,Secret,2.38,8,F,F,F,Password 36660,4beb1f7a,GitHub,472d4c24,data/472d4c24/test/4beb1f7a.rb,88,88,Template,T,22,31,F,F,CharsOnly,,,Secret,2.73,9,F,F,F,Password 36661,4beb1f7a,GitHub,472d4c24,data/472d4c24/test/4beb1f7a.rb,103,103,Template,T,22,31,F,F,CharsOnly,,,Secret,2.73,9,F,F,F,Password diff --git a/meta/48fd3902.csv b/meta/48fd3902.csv index 4b1f8acf..a451f651 100644 --- a/meta/48fd3902.csv +++ b/meta/48fd3902.csv @@ -141,7 +141,7 @@ Id,FileID,Domain,RepoName,FilePath,LineStart,LineEnd,GroundTruth,WithWords,Value 86784,a73bbe0d,GitHub,48fd3902,data/48fd3902/test/a73bbe0d.java,760,760,T,F,70,80,F,F,,,,,3.12,10,F,F,F,Password:Credential 86785,aaa8197f,GitHub,48fd3902,data/48fd3902/test/aaa8197f.kt,290,290,F,F,66,79,F,F,,,,,2.9,13,F,F,F,Password 86786,aaa8197f,GitHub,48fd3902,data/48fd3902/test/aaa8197f.kt,336,336,F,F,29,42,F,F,,,,,2.9,13,F,F,F,Password -86787,b169997c,GitHub,48fd3902,data/48fd3902/test/b169997c.java,62,62,T,F,54,74,F,F,,,,,3.03,20,F,F,F,Password +86787,b169997c,GitHub,48fd3902,data/48fd3902/test/b169997c.java,62,62,F,F,54,74,F,F,,,,,3.03,20,F,F,F,Password 86788,c370d8f8,GitHub,48fd3902,data/48fd3902/test/c370d8f8.kt,759,759,F,F,24,37,F,F,,,,,2.9,13,F,F,F,Password 86789,c370d8f8,GitHub,48fd3902,data/48fd3902/test/c370d8f8.kt,1308,1308,F,F,39,54,F,F,,,,,3.06,15,F,F,F,Password 86790,c370d8f8,GitHub,48fd3902,data/48fd3902/test/c370d8f8.kt,1086,1086,F,F,41,45,F,F,,,,,2.0,4,F,F,F,Key @@ -178,7 +178,7 @@ Id,FileID,Domain,RepoName,FilePath,LineStart,LineEnd,GroundTruth,WithWords,Value 109452,1be4753a,GitHub,48fd3902,data/48fd3902/test/1be4753a.java,42,42,F,F,16,36,F,F,,,,,4.3685226,-1,F,F,F,Password 109456,379ffa27,GitHub,48fd3902,data/48fd3902/test/379ffa27.key,1,54,T,F,0,0,F,F,,,,,6.047873,,F,F,F,Certificate 109458,41abbdc5,GitHub,48fd3902,data/48fd3902/test/41abbdc5.java,100,100,T,F,29,38,F,F,,,,,3.0957952,-1,F,F,F,Password -109461,6e09cb2b,GitHub,48fd3902,data/48fd3902/test/6e09cb2b.java,24,24,T,F,33,47,F,F,,,,,3.5,-1,F,F,F,Password +109461,6e09cb2b,GitHub,48fd3902,data/48fd3902/test/6e09cb2b.java,24,24,F,F,33,47,F,F,,,,,3.5,-1,F,F,F,Password 109477,9417bc40,GitHub,48fd3902,data/48fd3902/test/9417bc40.java,94,94,F,F,34,47,F,F,,,,,3.5068905,-1,F,F,F,Password 109480,c6175528,GitHub,48fd3902,data/48fd3902/test/c6175528.kt,215,241,T,F,12,83,F,F,,,,,5.6877875,-1,F,F,F,Certificate 109481,c6175528,GitHub,48fd3902,data/48fd3902/test/c6175528.kt,105,105,F,F,31,50,F,F,,,,,3.6304126,-1,F,F,F,Password diff --git a/meta/57c424f8.csv b/meta/57c424f8.csv index 3e9879ae..48855cb0 100644 --- a/meta/57c424f8.csv +++ b/meta/57c424f8.csv @@ -7,7 +7,7 @@ Id,FileID,Domain,RepoName,FilePath,LineStart,LineEnd,GroundTruth,WithWords,Value 5745,b0840f0e,GitHub,57c424f8,data/57c424f8/test/b0840f0e.java,81,81,F,F,-1,-1,F,F,,,,,0.0,0,F,F,F,Key 9070,07bf921e,GitHub,57c424f8,data/57c424f8/src/07bf921e.java,40,40,F,F,-1,-1,F,F,,,,,0.0,0,F,F,F,Key 11466,03e234ba,GitHub,57c424f8,data/57c424f8/src/03e234ba.java,100,100,F,F,-1,-1,F,F,,,,,0.0,0,F,F,F,Key -12718,b5b3fa31,GitHub,57c424f8,data/57c424f8/src/b5b3fa31.yml,79,79,F,F,-1,-1,F,F,,,,,0.0,0,F,F,F,Key +12718,b5b3fa31,GitHub,57c424f8,data/57c424f8/src/b5b3fa31.yml,79,79,T,F,11,27,F,F,,,,,0.0,0,F,F,F,Key 13424,4053b8df,GitHub,57c424f8,data/57c424f8/src/4053b8df.java,40,40,F,F,,,F,F,,,,,0,0,F,F,F,Secret:Key 14122,55cf96a7,GitHub,57c424f8,data/57c424f8/src/55cf96a7.yml,26,26,F,F,,,F,F,,,,,0,0,F,F,F,Password 14540,a40cbd6a,GitHub,57c424f8,data/57c424f8/src/a40cbd6a.java,50,50,F,F,-1,-1,F,F,,,,,0.0,0,F,F,F,Key @@ -32,7 +32,7 @@ Id,FileID,Domain,RepoName,FilePath,LineStart,LineEnd,GroundTruth,WithWords,Value 34291,2c698a15,GitHub,57c424f8,data/57c424f8/test/2c698a15.java,37,37,T,F,40,168,F,F,Any,,JSON Web Token,Token,5.31,128,F,F,F,JSON Web Token:Token 47478,26e016da,GitHub,57c424f8,data/57c424f8/test/26e016da.java,174,174,T,F,31,63,F,F,Any,,,Auth,3.65,32,F,F,F,Bitbucket Client ID:Bitbucket Client Secret 48051,bddef9f5,GitHub,57c424f8,data/57c424f8/test/bddef9f5.java,60,60,T,F,35,67,F,F,Any,,,Unknown,3.6,32,F,F,F,Key -74143,b5b3fa31,GitHub,57c424f8,data/57c424f8/src/b5b3fa31.yml,89,89,F,F,-1,-1,F,F,,,,,0.0,0,F,F,F,Key +74143,b5b3fa31,GitHub,57c424f8,data/57c424f8/src/b5b3fa31.yml,89,89,T,F,9,25,F,F,,,,,0.0,0,F,F,F,Key 108438,f23d7ddd,GitHub,57c424f8,data/57c424f8/test/f23d7ddd.java,63,63,F,F,,,F,F,,,,,0.00,,F,F,F,Secret 108439,14ae04f1,GitHub,57c424f8,data/57c424f8/test/14ae04f1.java,52,52,F,F,,,F,F,,,,,0.00,,F,F,F,Secret 108446,26e016da,GitHub,57c424f8,data/57c424f8/test/26e016da.java,132,132,T,F,72,104,F,F,Any,,,Secret,3.9,32,F,F,F,Secret diff --git a/meta/680ef0c1.csv b/meta/680ef0c1.csv index a8f19f72..aabe63d7 100644 --- a/meta/680ef0c1.csv +++ b/meta/680ef0c1.csv @@ -31,7 +31,7 @@ Id,FileID,Domain,RepoName,FilePath,LineStart,LineEnd,GroundTruth,WithWords,Value 14717,8a0a4f9a,GitHub,680ef0c1,data/680ef0c1/test/8a0a4f9a.go,359,359,F,F,-1,-1,F,F,,,,,0.0,0,F,F,F,Key 14718,8a0a4f9a,GitHub,680ef0c1,data/680ef0c1/test/8a0a4f9a.go,123,123,F,F,-1,-1,F,F,,,,,0.0,0,F,F,F,Key 14719,8a0a4f9a,GitHub,680ef0c1,data/680ef0c1/test/8a0a4f9a.go,1098,1098,F,F,-1,-1,F,F,,,,,0.0,0,F,F,F,Key -16972,879d7ff8,GitHub,680ef0c1,data/680ef0c1/src/879d7ff8.go,1015,1015,F,F,,,F,F,,,,,0,0,F,F,F,Password +16972,879d7ff8,GitHub,680ef0c1,data/680ef0c1/src/879d7ff8.go,1015,1015,T,F,17,23,F,F,,,,,0,0,F,F,F,Password 24830,e684bdad,GitHub,680ef0c1,data/680ef0c1/src/e684bdad.go,100,100,F,F,-1,-1,F,F,,,,,0.0,0,F,F,F,Key 24836,e684bdad,GitHub,680ef0c1,data/680ef0c1/src/e684bdad.go,192,192,F,F,-1,-1,F,F,,,,,0.0,0,F,F,F,Key 25045,27a244b7,GitHub,680ef0c1,data/680ef0c1/test/27a244b7.go,143,143,F,F,-1,-1,F,F,,,,,0.0,0,F,F,F,Key diff --git a/meta/90aebe4a.csv b/meta/90aebe4a.csv index 79086a8e..eabd8785 100644 --- a/meta/90aebe4a.csv +++ b/meta/90aebe4a.csv @@ -88,3 +88,4 @@ Id,FileID,Domain,RepoName,FilePath,LineStart,LineEnd,GroundTruth,WithWords,Value 130896,38c3f879,GitHub,90aebe4a,data/90aebe4a/test/38c3f879.js,107,107,F,F,,,F,F,,,,,0.00,,F,F,F,Password 130899,f5a78215,GitHub,90aebe4a,data/90aebe4a/src/f5a78215.js,40,40,F,F,,,F,F,,,,,0.00,,F,F,F,Password 130901,13bd9d7a,GitHub,90aebe4a,data/90aebe4a/other/13bd9d7a,48,48,F,F,,,F,F,,,,,0.00,,F,F,F,Password +1479610,fc63a614,GitHub,90aebe4a,data/90aebe4a/test/fc63a614.js,230,230,T,F,17,26,F,F,,,,,0.0,0,F,F,F,Password diff --git a/meta/a15774b8.csv b/meta/a15774b8.csv index 8fd446ef..c5212cbd 100644 --- a/meta/a15774b8.csv +++ b/meta/a15774b8.csv @@ -226,7 +226,7 @@ Id,FileID,Domain,RepoName,FilePath,LineStart,LineEnd,GroundTruth,WithWords,Value 34137,5a34df94,GitHub,a15774b8,data/a15774b8/src/5a34df94.yml,23,23,Template,T,33,37,F,F,CharsOnly,,,Secret,1.50,4,F,F,F,Password 34156,aa6a6ccc,GitHub,a15774b8,data/a15774b8/src/aa6a6ccc.rsp,495,495,Template,T,10,18,F,F,CharsOnly,,,Secret,2.75,8,F,F,F,Password 34187,eed3b7e7,GitHub,a15774b8,data/a15774b8/src/eed3b7e7.py,388,388,T,T,21,29,F,F,Any,,,Secret,2.75,8,F,F,F,Password -34259,a2cc6019,GitHub,a15774b8,data/a15774b8/other/a2cc6019,21,21,Template,T,69,77,F,F,Any,,,Secret,2.46,10,F,F,F,Password +34259,a2cc6019,GitHub,a15774b8,data/a15774b8/other/a2cc6019,21,21,T,T,69,77,F,F,Any,,,Secret,2.46,10,F,F,F,Password 34502,fa46a758,GitHub,a15774b8,data/a15774b8/src/fa46a758.py,249,249,T,T,22,30,F,F,Any,,,Secret,2.75,8,F,F,F,Password 34516,fa46a758,GitHub,a15774b8,data/a15774b8/src/fa46a758.py,242,242,T,T,19,27,F,F,Any,,,Secret,2.75,8,F,F,F,Password 34547,47d6ec60,GitHub,a15774b8,data/a15774b8/other/47d6ec60.md,31,31,T,T,17,32,F,F,CharsOnly,,,Secret,3.37,15,F,F,F,Dropbox App secret:Password diff --git a/meta/b6b2487d.csv b/meta/b6b2487d.csv index fbea9f4d..59a0f611 100644 --- a/meta/b6b2487d.csv +++ b/meta/b6b2487d.csv @@ -463,7 +463,7 @@ Id,FileID,Domain,RepoName,FilePath,LineStart,LineEnd,GroundTruth,WithWords,Value 22533,538c8c8b,GitHub,b6b2487d,data/b6b2487d/src/538c8c8b.json,279,279,F,F,,,F,F,,,,,0,0,F,F,F,Password 22534,a0e089f7,GitHub,b6b2487d,data/b6b2487d/src/a0e089f7.json,304,304,F,F,,,F,F,,,,,0,0,F,F,F,Password 22535,46be83f8,GitHub,b6b2487d,data/b6b2487d/src/46be83f8.json,304,304,F,F,,,F,F,,,,,0,0,F,F,F,Password -22536,e96794ab,GitHub,b6b2487d,data/b6b2487d/src/e96794ab.json,254,254,F,F,,,F,F,,,,,0,0,F,F,F,Password +22536,e96794ab,GitHub,b6b2487d,data/b6b2487d/src/e96794ab.json,254,254,T,F,25,42,F,F,,,,,0,0,F,F,F,Password 22537,d4fbedce,GitHub,b6b2487d,data/b6b2487d/src/d4fbedce.json,318,318,F,F,,,F,F,,,,,0,0,F,F,F,Password 22538,f7f6b259,GitHub,b6b2487d,data/b6b2487d/src/f7f6b259.json,269,269,F,F,,,F,F,,,,,0,0,F,F,F,Password 22539,55e839d8,GitHub,b6b2487d,data/b6b2487d/src/55e839d8.json,281,281,F,F,,,F,F,,,,,0,0,F,F,F,Password diff --git a/meta/bbb4193f.csv b/meta/bbb4193f.csv index 2ae07a27..3fffc64a 100644 --- a/meta/bbb4193f.csv +++ b/meta/bbb4193f.csv @@ -41,13 +41,13 @@ Id,FileID,Domain,RepoName,FilePath,LineStart,LineEnd,GroundTruth,WithWords,Value 26141,b496d6b7,GitHub,bbb4193f,data/bbb4193f/src/b496d6b7.vue,87,87,F,F,-1,-1,F,F,,,,,0.0,0,F,F,F,Key 26142,b496d6b7,GitHub,bbb4193f,data/bbb4193f/src/b496d6b7.vue,6,6,F,F,-1,-1,F,F,,,,,0.0,0,F,F,F,Key 26143,6bd12b8f,GitHub,bbb4193f,data/bbb4193f/src/6bd12b8f.vue,6,6,F,F,-1,-1,F,F,,,,,0.0,0,F,F,F,Key -29385,fa8e378d,GitHub,bbb4193f,data/bbb4193f/src/fa8e378d.properties,38,38,F,F,29,157,F,F,,,,,0,0,F,F,F,Password +29385,fa8e378d,GitHub,bbb4193f,data/bbb4193f/src/fa8e378d.properties,38,38,F,F,29,157,F,F,,,,,0,0,F,F,F,Password:Key 29388,83ab6bcf,GitHub,bbb4193f,data/bbb4193f/test/83ab6bcf.java,30,30,Template,F,40,43,F,F,CharsOnly,,,Secret,0.92,3,F,F,F,Password 29411,19a76a5a,GitHub,bbb4193f,data/bbb4193f/test/19a76a5a.properties,16,16,Template,F,22,28,F,F,NumbersOnly,,,Secret,2.58,6,F,F,F,Password 29821,fffbc0b2,GitHub,bbb4193f,data/bbb4193f/test/fffbc0b2.java,26,26,Template,F,49,54,F,F,CharsOnly,,,Secret,0.0,5,F,F,F,Password 30035,3f7ea830,GitHub,bbb4193f,data/bbb4193f/test/3f7ea830.java,11,11,T,T,56,65,F,F,Any,,,Secret,2.95,9,F,F,F,Password 30520,f20d0b36,GitHub,bbb4193f,data/bbb4193f/src/f20d0b36.properties,18,18,Template,F,21,61,F,F,Any,,,Secret,3.81,40,F,F,F,Password -30521,a979c545,GitHub,bbb4193f,data/bbb4193f/test/a979c545.properties,31,31,F,F,28,156,F,F,,,,,0,0,F,F,F,Password +30521,a979c545,GitHub,bbb4193f,data/bbb4193f/test/a979c545.properties,31,31,F,F,28,156,F,F,,,,,0,0,F,F,F,Password:Key 31157,b64fb722,GitHub,bbb4193f,data/bbb4193f/src/b64fb722.sh,94,94,T,T,24,33,F,F,Any,,,Secret,2.95,9,F,F,F,Password 32171,19a76a5a,GitHub,bbb4193f,data/bbb4193f/test/19a76a5a.properties,8,8,T,T,16,21,F,F,CharsOnly,,,Secret,1.92,5,F,F,F,Password 32299,f20d0b36,GitHub,bbb4193f,data/bbb4193f/src/f20d0b36.properties,90,90,T,T,33,38,F,F,CharsOnly,,,Secret,1.92,5,F,F,F,Password diff --git a/meta/c2d95db7.csv b/meta/c2d95db7.csv index 73ec76eb..d7390ab3 100644 --- a/meta/c2d95db7.csv +++ b/meta/c2d95db7.csv @@ -38,7 +38,7 @@ Id,FileID,Domain,RepoName,FilePath,LineStart,LineEnd,GroundTruth,WithWords,Value 29384,a3046da0,GitHub,c2d95db7,data/c2d95db7/test/a3046da0.example,638,638,Template,T,14,27,F,F,Any,,,Unknown,2.61,13,F,F,F,Key 29396,be5fd676,GitHub,c2d95db7,data/c2d95db7/src/be5fd676.conf,428,428,Template,T,33,39,F,F,CharsOnly,,,Secret,2.25,6,F,F,F,Auth:Password 29769,be5fd676,GitHub,c2d95db7,data/c2d95db7/src/be5fd676.conf,55,55,T,T,18,47,F,F,Any,,,Secret,3.67,29,F,F,F,Password:Secret -29785,8e165a9d,GitHub,c2d95db7,data/c2d95db7/src/8e165a9d.sh,371,371,F,F,18,38,F,F,,,,,0.0,0,F,F,F,Password +29785,8e165a9d,GitHub,c2d95db7,data/c2d95db7/src/8e165a9d.sh,371,371,T,F,18,38,F,F,,,,,0.0,0,F,F,F,Password 30241,a3046da0,GitHub,c2d95db7,data/c2d95db7/test/a3046da0.example,802,802,Template,T,29,57,F,F,Any,,,Unknown,3.15,28,F,F,F,Key 30487,a3046da0,GitHub,c2d95db7,data/c2d95db7/test/a3046da0.example,414,414,Template,T,17,23,F,F,CharsOnly,,,Secret,2.25,6,F,F,F,Password 30491,a3046da0,GitHub,c2d95db7,data/c2d95db7/test/a3046da0.example,814,814,T,T,17,33,F,F,CharsOnly,,,Secret,2.75,16,F,F,F,Key:Secret @@ -56,7 +56,7 @@ Id,FileID,Domain,RepoName,FilePath,LineStart,LineEnd,GroundTruth,WithWords,Value 31173,e45e45ba,GitHub,c2d95db7,data/c2d95db7/src/e45e45ba.yml,472,472,Template,T,27,33,F,F,CharsOnly,,,Unknown,1.92,6,F,F,F,Key 31275,7ccdfb29,GitHub,c2d95db7,data/c2d95db7/other/7ccdfb29.md,991,991,Template,T,37,54,F,F,Any,,,Token,2.77,17,F,F,F,Token 31277,a3046da0,GitHub,c2d95db7,data/c2d95db7/test/a3046da0.example,740,740,Template,T,34,57,F,F,Any,,,Token,3.10,23,F,F,F,Token -31286,a3046da0,GitHub,c2d95db7,data/c2d95db7/test/a3046da0.example,735,735,Template,T,25,40,F,F,Any,,,Secret,3.20,15,F,F,F,Password +31286,a3046da0,GitHub,c2d95db7,data/c2d95db7/test/a3046da0.example,735,735,T,T,25,40,F,F,Any,,,Secret,3.20,15,F,F,F,Password 31376,7ccdfb29,GitHub,c2d95db7,data/c2d95db7/other/7ccdfb29.md,1254,1254,Template,T,24,33,F,F,CharsOnly,,,Secret,2.64,9,F,F,F,Key:Secret 31385,a3046da0,GitHub,c2d95db7,data/c2d95db7/test/a3046da0.example,579,579,Template,T,24,38,F,F,Any,,,Token,2.51,14,F,F,F,Token 31388,a3046da0,GitHub,c2d95db7,data/c2d95db7/test/a3046da0.example,577,577,Template,T,24,38,F,F,Any,,,Token,2.56,14,F,F,F,Token diff --git a/meta/f710ac3c.csv b/meta/f710ac3c.csv index 82b16e6e..8819a0cf 100644 --- a/meta/f710ac3c.csv +++ b/meta/f710ac3c.csv @@ -61,7 +61,7 @@ Id,FileID,Domain,RepoName,FilePath,LineStart,LineEnd,GroundTruth,WithWords,Value 26839,56e55b10,GitHub,f710ac3c,data/f710ac3c/src/56e55b10.txt,1527,1527,F,F,,,F,F,,,,,0,0,F,F,F,Password 27218,16f1d758,GitHub,f710ac3c,data/f710ac3c/test/16f1d758.py,235,235,Template,F,86,92,F,F,CharsOnly,,,Secret,2.58,6,F,F,F,Password 27395,16f1d758,GitHub,f710ac3c,data/f710ac3c/test/16f1d758.py,173,173,Template,F,62,68,F,F,CharsOnly,,,Secret,2.58,6,F,F,F,Password -27598,8d982008,GitHub,f710ac3c,data/f710ac3c/test/8d982008.py,199,199,F,T,37,52,F,F,Any,,,Secret,2.93,15,F,F,F,Password +27598,8d982008,GitHub,f710ac3c,data/f710ac3c/test/8d982008.py,199,199,T,T,37,52,F,F,Any,,,Secret,2.93,15,F,F,F,Password 27635,16f1d758,GitHub,f710ac3c,data/f710ac3c/test/16f1d758.py,223,223,Template,F,64,70,F,F,CharsOnly,,,Secret,2.58,6,F,F,F,Password 28170,142d4e03,GitHub,f710ac3c,data/f710ac3c/test/142d4e03.py,40,40,T,F,76,83,F,F,Any,,,Secret,2.12,7,F,F,F,Password 28171,16f1d758,GitHub,f710ac3c,data/f710ac3c/test/16f1d758.py,149,149,Template,F,70,76,F,F,CharsOnly,,,Secret,2.58,6,F,F,F,Password From ca1d8e811eea74407555436dd4b9cf075a8d32b5 Mon Sep 17 00:00:00 2001 From: Roman Babenko Date: Thu, 5 Sep 2024 17:18:06 +0300 Subject: [PATCH 16/49] T/F new markup --- meta/0f133e09.csv | 1 + meta/81cd05d0.csv | 2 ++ 2 files changed, 3 insertions(+) diff --git a/meta/0f133e09.csv b/meta/0f133e09.csv index ef0ae2b7..c28e2179 100644 --- a/meta/0f133e09.csv +++ b/meta/0f133e09.csv @@ -1769,3 +1769,4 @@ Id,FileID,Domain,RepoName,FilePath,LineStart,LineEnd,GroundTruth,WithWords,Value 1351520,c7b13616,GitHub,0f133e09,data/0f133e09/test/c7b13616.txt,9,9,T,F,1187,1249,F,F,,,,,0.0,0,F,F,F,AWS S3 Bucket 1351521,c7b13616,GitHub,0f133e09,data/0f133e09/test/c7b13616.txt,9,9,T,F,1250,1302,F,F,,,,,0.0,0,F,F,F,AWS S3 Bucket 1351522,c7b13616,GitHub,0f133e09,data/0f133e09/test/c7b13616.txt,9,9,T,F,1584,1613,F,F,,,,,0.0,0,F,F,F,AWS S3 Bucket +1479613,03c77d4f,GitHub,0f133e09,data/0f133e09/src/03c77d4f.py,314,314,F,F,49,54,F,F,,,,,0.0,0,F,F,F,Auth diff --git a/meta/81cd05d0.csv b/meta/81cd05d0.csv index ca70d2d4..cdd3baa2 100644 --- a/meta/81cd05d0.csv +++ b/meta/81cd05d0.csv @@ -5250,3 +5250,5 @@ Id,FileID,Domain,RepoName,FilePath,LineStart,LineEnd,GroundTruth,WithWords,Value 1341485,a84a47e9,GitHub,81cd05d0,data/81cd05d0/src/a84a47e9.json,198563,198563,T,F,1074,1110,F,F,,,,,0.0,0,F,F,F,UUID:Token 1341486,f02052a6,GitHub,81cd05d0,data/81cd05d0/src/f02052a6.json,3724,3724,T,F,1072,1108,F,F,,,,,0.0,0,F,F,F,UUID:Token 1341487,f02052a6,GitHub,81cd05d0,data/81cd05d0/src/f02052a6.json,3761,3761,T,F,1148,1184,F,F,,,,,0.0,0,F,F,F,UUID:Token +1479614,334b19eb,GitHub,81cd05d0,data/81cd05d0/src/334b19eb.json,32577,32577,T,F,28,,F,F,,,,,0.0,0,F,F,F,Secret +1479615,334b19eb,GitHub,81cd05d0,data/81cd05d0/src/334b19eb.json,32790,32790,T,F,28,,F,F,,,,,0.0,0,F,F,F,Secret From 7204ae12146b853a108a889ec76c4eed1bb84fd8 Mon Sep 17 00:00:00 2001 From: Roman Babenko Date: Thu, 5 Sep 2024 17:25:18 +0300 Subject: [PATCH 17/49] cibmfix --- .ci/benchmark.txt | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.ci/benchmark.txt b/.ci/benchmark.txt index 665ccc80..bc76b256 100644 --- a/.ci/benchmark.txt +++ b/.ci/benchmark.txt @@ -1,6 +1,6 @@ -META MD5 f8ea5ee867e628f3add2f09d730f72df +META MD5 42c914c398d140a6fa908f0955bdad01 DATA MD5 79ac8163a2132e6d6588718e81ced074 -DATA: 16345596 interested lines. MARKUP: 62812 items +DATA: 16345596 interested lines. MARKUP: 62813 items FileType FileNumber ValidLines Positives Negatives Templates --------------- ------------ ------------ ----------- ----------- ----------- 194 28318 71 417 90 @@ -86,7 +86,7 @@ FileType FileNumber ValidLines Positives Negatives Templat .jenkinsfile 1 58 2 6 .jinja2 1 64 2 .js 659 536413 536 2496 328 -.json 851 13046493 1072 10910 140 +.json 851 13046493 1074 10910 140 .jsp 13 3202 1 40 .jsx 7 857 19 .jwt 1 1 2 @@ -153,7 +153,7 @@ FileType FileNumber ValidLines Positives Negatives Templat .pug 2 193 2 .purs 1 69 4 .pxd 1 150 5 2 -.py 890 291553 684 3300 723 +.py 890 291553 684 3301 723 .pyi 4 1361 9 .pyp 1 167 1 .pyx 2 1094 23 @@ -223,7 +223,7 @@ FileType FileNumber ValidLines Positives Negatives Templat .yml 419 36169 555 888 378 .zsh 6 872 12 .zsh-theme 1 97 1 -TOTAL: 10264 16345596 12202 50508 5098 +TOTAL: 10264 16345596 12204 50509 5098 credsweeper result_cnt : 0, lost_cnt : 0, true_cnt : 0, false_cnt : 0 Rules Positives Negatives Templates Reported TP FP TN FN FPR FNR ACC PRC RCL F1 ------------------------------ ----------- ----------- ----------- ---------- ---- ---- ----- ----- -------- -------- -------- ----- -------- ---- @@ -232,7 +232,7 @@ AWS Client ID 168 21 0 AWS Multi 75 16 0 0 0 16 75 0.000000 1.000000 0.175824 0.000000 AWS S3 Bucket 66 24 0 0 0 24 66 0.000000 1.000000 0.266667 0.000000 Atlassian Old PAT token 27 308 3 0 0 311 27 0.000000 1.000000 0.920118 0.000000 -Auth 419 2738 76 0 0 2814 419 0.000000 1.000000 0.870399 0.000000 +Auth 419 2739 76 0 0 2815 419 0.000000 1.000000 0.870439 0.000000 Azure Access Token 19 0 0 0 0 0 19 1.000000 0.000000 0.000000 BASE64 Private Key 7 4 0 0 0 4 7 0.000000 1.000000 0.363636 0.000000 BASE64 encoded PEM Private Key 7 0 0 0 0 0 7 1.000000 0.000000 0.000000 @@ -264,11 +264,11 @@ Other 0 8291 1 PEM Private Key 1019 1483 0 0 0 1483 1019 0.000000 1.000000 0.592726 0.000000 Password 1854 7537 2698 0 0 10235 1854 0.000000 1.000000 0.846637 0.000000 Salt 45 76 2 0 0 78 45 0.000000 1.000000 0.634146 0.000000 -Secret 1298 1576 798 0 0 2374 1298 0.000000 1.000000 0.646514 0.000000 +Secret 1300 1576 798 0 0 2374 1300 0.000000 1.000000 0.646162 0.000000 Seed 1 6 0 0 0 6 1 0.000000 1.000000 0.857143 0.000000 Slack Token 4 1 0 0 0 1 4 0.000000 1.000000 0.200000 0.000000 Token 652 4171 438 0 0 4609 652 0.000000 1.000000 0.876069 0.000000 Twilio API Key 0 5 2 0 0 7 0 0.000000 1.000000 URL Credentials 210 151 220 0 0 371 210 0.000000 1.000000 0.638554 0.000000 UUID 1069 265 0 0 0 265 1069 0.000000 1.000000 0.198651 0.000000 - 12202 50508 5098 0 0 0 50508 12202 0.000000 1.000000 0.805422 0.000000 + 12204 50509 5098 0 0 0 50509 12204 0.000000 1.000000 0.805399 0.000000 From 24c91986c7279ab73f8d0be4bf59635cb9cf4416 Mon Sep 17 00:00:00 2001 From: Roman Babenko Date: Thu, 5 Sep 2024 18:08:53 +0300 Subject: [PATCH 18/49] pos correction --- .ci/benchmark.txt | 2 +- meta/6c73b80a.csv | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.ci/benchmark.txt b/.ci/benchmark.txt index bc76b256..24abccfa 100644 --- a/.ci/benchmark.txt +++ b/.ci/benchmark.txt @@ -1,4 +1,4 @@ -META MD5 42c914c398d140a6fa908f0955bdad01 +META MD5 78cb5d3d6472d88c6f785225321626ea DATA MD5 79ac8163a2132e6d6588718e81ced074 DATA: 16345596 interested lines. MARKUP: 62813 items FileType FileNumber ValidLines Positives Negatives Templates diff --git a/meta/6c73b80a.csv b/meta/6c73b80a.csv index fcf1c1af..231dd73f 100644 --- a/meta/6c73b80a.csv +++ b/meta/6c73b80a.csv @@ -428,7 +428,7 @@ Id,FileID,Domain,RepoName,FilePath,LineStart,LineEnd,GroundTruth,WithWords,Value 37426,88763956,GitHub,6c73b80a,data/6c73b80a/test/88763956.go,3561,3561,T,T,32,36,F,F,Any,,,Secret,1.46,5,F,F,F,Password 37429,515f4ca6,GitHub,6c73b80a,data/6c73b80a/test/515f4ca6.go,4412,4412,Template,F,26,29,F,F,Any,,,Secret,1.58,3,F,F,F,Password 37430,515f4ca6,GitHub,6c73b80a,data/6c73b80a/test/515f4ca6.go,4434,4434,Template,F,26,29,F,F,Any,,,Secret,1.58,3,F,F,F,Password -37439,515f4ca6,GitHub,6c73b80a,data/6c73b80a/test/515f4ca6.go,2666,2666,F,F,26,30,F,F,Any,,,Short,1.5,4,F,F,F,Password +37439,515f4ca6,GitHub,6c73b80a,data/6c73b80a/test/515f4ca6.go,2666,2666,F,F,26,29,F,F,Any,,,Short,1.5,4,F,F,F,Password 37440,515f4ca6,GitHub,6c73b80a,data/6c73b80a/test/515f4ca6.go,2660,2660,T,T,28,33,F,F,Any,,,Secret,1.82,6,F,F,F,Password 37441,633f9ac2,GitHub,6c73b80a,data/6c73b80a/test/633f9ac2.go,3673,3673,Template,F,37,40,F,F,Any,,,Secret,1.58,3,F,F,F,Password 37442,633f9ac2,GitHub,6c73b80a,data/6c73b80a/test/633f9ac2.go,3708,3708,Template,F,37,40,F,F,Any,,,Secret,1.58,3,F,F,F,Password From 89ec0cd9fb0be66278a858904bb6783c07a54603 Mon Sep 17 00:00:00 2001 From: Roman Babenko Date: Fri, 6 Sep 2024 10:17:35 +0300 Subject: [PATCH 19/49] [no ci] F for public key --- meta/64c97cb8.csv | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/meta/64c97cb8.csv b/meta/64c97cb8.csv index 1e16594b..30bfcea5 100644 --- a/meta/64c97cb8.csv +++ b/meta/64c97cb8.csv @@ -100,7 +100,7 @@ Id,FileID,Domain,RepoName,FilePath,LineStart,LineEnd,GroundTruth,WithWords,Value 97076,e57a6dac,GitHub,64c97cb8,data/64c97cb8/test/e57a6dac.cs,71,71,T,F,28,37,F,F,,,,,2.73,9,F,F,F,Password 97077,e57a6dac,GitHub,64c97cb8,data/64c97cb8/test/e57a6dac.cs,37,37,T,F,28,37,F,F,,,,,2.37,9,F,F,F,Password 97078,f8402747,GitHub,64c97cb8,data/64c97cb8/test/f8402747.cs,30,30,T,F,34,47,F,F,,,,,3.11,13,F,F,F,Password -109489,37697095,GitHub,64c97cb8,data/64c97cb8/src/37697095.config,4,4,T,F,145,161,F,F,,,,,3.625,-1,F,F,F,Token:Key +109489,37697095,GitHub,64c97cb8,data/64c97cb8/src/37697095.config,4,4,F,F,145,161,F,F,,,,,3.625,-1,F,F,F,Token:Key 109490,a73347eb,GitHub,64c97cb8,data/64c97cb8/src/a73347eb.cs,40,40,F,F,48,56,F,F,,,,,2.9219282,-1,F,F,F,Password 1113138,79ebb761,GitHub,64c97cb8,data/64c97cb8/src/79ebb761.cs,462,462,F,F,45,57,F,F,,,,,0.0,0,F,F,F,Credential 1113142,f8402747,GitHub,64c97cb8,data/64c97cb8/test/f8402747.cs,16,16,T,F,34,47,F,F,,,,,0.0,0,F,F,F,Password From 90b771d3c1a91523e5e676c90cc5124ee69d1073 Mon Sep 17 00:00:00 2001 From: Roman Babenko Date: Fri, 6 Sep 2024 10:19:16 +0300 Subject: [PATCH 20/49] BM --- .ci/benchmark.txt | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.ci/benchmark.txt b/.ci/benchmark.txt index 24abccfa..88390a7e 100644 --- a/.ci/benchmark.txt +++ b/.ci/benchmark.txt @@ -1,5 +1,5 @@ -META MD5 78cb5d3d6472d88c6f785225321626ea -DATA MD5 79ac8163a2132e6d6588718e81ced074 +META MD5 a482f0f5332d75a1d014557096beb9ae +DATA MD5 d02f5c7efa56379178bb6e7e19e61ebd DATA: 16345596 interested lines. MARKUP: 62813 items FileType FileNumber ValidLines Positives Negatives Templates --------------- ------------ ------------ ----------- ----------- ----------- @@ -30,7 +30,7 @@ FileType FileNumber ValidLines Positives Negatives Templat .cnf 8 858 15 36 16 .coffee 1 585 2 .conf 60 4945 55 64 53 -.config 20 492 17 38 1 +.config 20 492 15 40 1 .cpp 15 5688 2 61 .creds 1 10 1 1 .crlf 1 27 1 @@ -223,7 +223,7 @@ FileType FileNumber ValidLines Positives Negatives Templat .yml 419 36169 555 888 378 .zsh 6 872 12 .zsh-theme 1 97 1 -TOTAL: 10264 16345596 12204 50509 5098 +TOTAL: 10264 16345596 12202 50511 5098 credsweeper result_cnt : 0, lost_cnt : 0, true_cnt : 0, false_cnt : 0 Rules Positives Negatives Templates Reported TP FP TN FN FPR FNR ACC PRC RCL F1 ------------------------------ ----------- ----------- ----------- ---------- ---- ---- ----- ----- -------- -------- -------- ----- -------- ---- @@ -258,7 +258,7 @@ Grafana Provisioned API Key 22 1 0 JSON Web Token 170 61 0 0 0 61 170 0.000000 1.000000 0.264069 0.000000 Jira / Confluence PAT token 0 4 0 0 0 4 0 0.000000 1.000000 Jira 2FA 15 6 1 0 0 7 15 0.000000 1.000000 0.318182 0.000000 -Key 3908 15720 482 0 0 16202 3908 0.000000 1.000000 0.805669 0.000000 +Key 3907 15721 482 0 0 16203 3907 0.000000 1.000000 0.805719 0.000000 Nonce 91 49 0 0 0 49 91 0.000000 1.000000 0.350000 0.000000 Other 0 8291 1 0 0 8292 0 0.000000 1.000000 PEM Private Key 1019 1483 0 0 0 1483 1019 0.000000 1.000000 0.592726 0.000000 @@ -267,8 +267,8 @@ Salt 45 76 2 Secret 1300 1576 798 0 0 2374 1300 0.000000 1.000000 0.646162 0.000000 Seed 1 6 0 0 0 6 1 0.000000 1.000000 0.857143 0.000000 Slack Token 4 1 0 0 0 1 4 0.000000 1.000000 0.200000 0.000000 -Token 652 4171 438 0 0 4609 652 0.000000 1.000000 0.876069 0.000000 +Token 651 4172 438 0 0 4610 651 0.000000 1.000000 0.876259 0.000000 Twilio API Key 0 5 2 0 0 7 0 0.000000 1.000000 URL Credentials 210 151 220 0 0 371 210 0.000000 1.000000 0.638554 0.000000 UUID 1069 265 0 0 0 265 1069 0.000000 1.000000 0.198651 0.000000 - 12204 50509 5098 0 0 0 50509 12204 0.000000 1.000000 0.805399 0.000000 + 12202 50511 5098 0 0 0 50511 12202 0.000000 1.000000 0.805431 0.000000 From a1062da6c3e27e0d144dc7a16cbb1903381ebc2d Mon Sep 17 00:00:00 2001 From: Roman Babenko Date: Fri, 6 Sep 2024 13:54:25 +0300 Subject: [PATCH 21/49] corrections --- meta/28728ab4.csv | 2 +- meta/2ba83c6a.csv | 2 +- meta/6c73b80a.csv | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/meta/28728ab4.csv b/meta/28728ab4.csv index 628dd4b5..24cd26b2 100644 --- a/meta/28728ab4.csv +++ b/meta/28728ab4.csv @@ -80,7 +80,7 @@ Id,FileID,Domain,RepoName,FilePath,LineStart,LineEnd,GroundTruth,WithWords,Value 10818,e8f39dd9,GitHub,28728ab4,data/28728ab4/src/e8f39dd9.go,77,77,F,F,,,F,F,,,,,0,0,F,F,F,API 11040,7eb207d4,GitHub,28728ab4,data/28728ab4/test/7eb207d4.go,20,20,F,F,,,F,F,,,,,0,0,F,F,F,Bitbucket Client Secret 12397,18282143,GitHub,28728ab4,data/28728ab4/other/18282143.md,65,65,F,F,-1,-1,F,F,,,,,0.0,0,F,F,F,Key -13141,56575ef0,GitHub,28728ab4,data/28728ab4/test/56575ef0.go,417,417,T,F,26,40,F,F,,,,,0.0,0,F,F,F,Password +13141,56575ef0,GitHub,28728ab4,data/28728ab4/test/56575ef0.go,417,417,T,F,26,40,F,F,,,,,0.0,0,F,F,F,Auth:Password 137298,56575ef0,GitHub,28728ab4,data/28728ab4/test/56575ef0.go,417,417,T,F,142,153,F,F,,,,,0.0,0,F,F,F,Auth 13600,b13659e2,GitHub,28728ab4,data/28728ab4/src/b13659e2.go,93,93,F,F,,,F,F,,,,,0,0,F,F,F,Auth 13601,b13659e2,GitHub,28728ab4,data/28728ab4/src/b13659e2.go,113,113,F,F,,,F,F,,,,,0,0,F,F,F,Token diff --git a/meta/2ba83c6a.csv b/meta/2ba83c6a.csv index d863c743..5f942ac1 100644 --- a/meta/2ba83c6a.csv +++ b/meta/2ba83c6a.csv @@ -9775,7 +9775,7 @@ Id,FileID,Domain,RepoName,FilePath,LineStart,LineEnd,GroundTruth,WithWords,Value 130793,cf577288,GitHub,2ba83c6a,data/2ba83c6a/src/cf577288.txt,55,55,F,F,,,F,F,,,,,0.00,,F,F,F,Password 130795,8a68cd28,GitHub,2ba83c6a,data/2ba83c6a/test/8a68cd28.txt,145,145,F,F,,,F,F,,,,,0.00,,F,F,F,Password 130797,c4df9321,GitHub,2ba83c6a,data/2ba83c6a/src/c4df9321.c,1343,1343,F,F,,,F,F,,,,,0.00,,F,F,F,Certificate -130799,07d8fbd6,GitHub,2ba83c6a,data/2ba83c6a/test/07d8fbd6.t,80,80,F,F,,,F,F,,,,,0.00,,F,F,F,Password +130799,07d8fbd6,GitHub,2ba83c6a,data/2ba83c6a/test/07d8fbd6.t,80,80,F,F,,,F,F,,,,,0.00,,F,F,F,Password:Salt 130800,d2be8070,GitHub,2ba83c6a,data/2ba83c6a/test/d2be8070.c,1209,1209,F,F,45,49,F,F,,,,,0.0,-1,F,F,F,Password 130801,0e0b1c0e,GitHub,2ba83c6a,data/2ba83c6a/src/0e0b1c0e.c,800,800,F,F,,,F,F,,,,,0.00,,F,F,F,Password 130802,eba027be,GitHub,2ba83c6a,data/2ba83c6a/src/eba027be.c,800,800,F,F,,,F,F,,,,,0.00,,F,F,F,Password diff --git a/meta/6c73b80a.csv b/meta/6c73b80a.csv index 231dd73f..d2a9362d 100644 --- a/meta/6c73b80a.csv +++ b/meta/6c73b80a.csv @@ -114,8 +114,8 @@ Id,FileID,Domain,RepoName,FilePath,LineStart,LineEnd,GroundTruth,WithWords,Value 26373,c8af9dd7,GitHub,6c73b80a,data/6c73b80a/src/c8af9dd7.go,882,882,F,F,,,F,F,,,,,0,0,F,F,F,Password 27330,1a13a17c,GitHub,6c73b80a,data/6c73b80a/test/1a13a17c.go,2837,2837,Template,F,36,39,F,F,Any,,,Secret,0.92,3,F,F,F,Password 27331,69abb428,GitHub,6c73b80a,data/6c73b80a/test/69abb428.go,10436,10436,Template,F,35,38,F,F,Any,,,Secret,0.92,3,F,F,F,Password -27332,1a13a17c,GitHub,6c73b80a,data/6c73b80a/test/1a13a17c.go,273,273,T,F,39,47,F,F,Any,,,Secret,2.0,8,F,F,F,Password -27334,1a13a17c,GitHub,6c73b80a,data/6c73b80a/test/1a13a17c.go,277,277,T,F,39,47,F,F,Any,,,Secret,2.0,8,F,F,F,Password +27332,1a13a17c,GitHub,6c73b80a,data/6c73b80a/test/1a13a17c.go,273,273,T,F,39,45,F,F,Any,,,Secret,2.0,8,F,F,F,Password +27334,1a13a17c,GitHub,6c73b80a,data/6c73b80a/test/1a13a17c.go,277,277,T,F,39,45,F,F,Any,,,Secret,2.0,8,F,F,F,Password 27344,1258dc58,GitHub,6c73b80a,data/6c73b80a/test/1258dc58.go,1799,1799,Template,T,38,46,F,F,Any,,,Secret,2.75,8,F,F,F,Password 27384,edc1c568,GitHub,6c73b80a,data/6c73b80a/test/edc1c568.go,920,920,Template,F,26,32,F,F,Any,,,Token,2.25,6,F,F,F,Token 27752,88763956,GitHub,6c73b80a,data/6c73b80a/test/88763956.go,592,592,T,T,47,52,F,F,Any,,,Secret,2.32,5,F,F,F,Password From 0dd65e959ab8614d60a34e50fe6cf52728b4dbef Mon Sep 17 00:00:00 2001 From: Roman Babenko Date: Fri, 6 Sep 2024 13:55:55 +0300 Subject: [PATCH 22/49] corrections --- .ci/benchmark.txt | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.ci/benchmark.txt b/.ci/benchmark.txt index 88390a7e..4402575a 100644 --- a/.ci/benchmark.txt +++ b/.ci/benchmark.txt @@ -1,4 +1,4 @@ -META MD5 a482f0f5332d75a1d014557096beb9ae +META MD5 744a34b5e241b09a9e84966b3e3b9bd0 DATA MD5 d02f5c7efa56379178bb6e7e19e61ebd DATA: 16345596 interested lines. MARKUP: 62813 items FileType FileNumber ValidLines Positives Negatives Templates @@ -63,7 +63,7 @@ FileType FileNumber ValidLines Positives Negatives Templat .gd 1 37 1 .gml 3 3075 16 .gni 3 5017 19 -.go 1080 566476 694 4123 738 +.go 1080 566476 695 4123 738 .golden 5 1168 1 13 29 .gradle 45 3265 4 90 100 .graphql 7 420 13 @@ -192,7 +192,7 @@ FileType FileNumber ValidLines Positives Negatives Templat .sublime-keymap 1 3 1 .sum 37 22854 283 .svg 1 638 12 -.t 9 1767 24 43 14 +.t 9 1767 24 44 14 .td 2 14002 6 .template 19 1633 4 35 11 .test 2 24 25 4 @@ -223,7 +223,7 @@ FileType FileNumber ValidLines Positives Negatives Templat .yml 419 36169 555 888 378 .zsh 6 872 12 .zsh-theme 1 97 1 -TOTAL: 10264 16345596 12202 50511 5098 +TOTAL: 10264 16345596 12203 50512 5098 credsweeper result_cnt : 0, lost_cnt : 0, true_cnt : 0, false_cnt : 0 Rules Positives Negatives Templates Reported TP FP TN FN FPR FNR ACC PRC RCL F1 ------------------------------ ----------- ----------- ----------- ---------- ---- ---- ----- ----- -------- -------- -------- ----- -------- ---- @@ -232,7 +232,7 @@ AWS Client ID 168 21 0 AWS Multi 75 16 0 0 0 16 75 0.000000 1.000000 0.175824 0.000000 AWS S3 Bucket 66 24 0 0 0 24 66 0.000000 1.000000 0.266667 0.000000 Atlassian Old PAT token 27 308 3 0 0 311 27 0.000000 1.000000 0.920118 0.000000 -Auth 419 2739 76 0 0 2815 419 0.000000 1.000000 0.870439 0.000000 +Auth 420 2739 76 0 0 2815 420 0.000000 1.000000 0.870170 0.000000 Azure Access Token 19 0 0 0 0 0 19 1.000000 0.000000 0.000000 BASE64 Private Key 7 4 0 0 0 4 7 0.000000 1.000000 0.363636 0.000000 BASE64 encoded PEM Private Key 7 0 0 0 0 0 7 1.000000 0.000000 0.000000 @@ -263,7 +263,7 @@ Nonce 91 49 0 Other 0 8291 1 0 0 8292 0 0.000000 1.000000 PEM Private Key 1019 1483 0 0 0 1483 1019 0.000000 1.000000 0.592726 0.000000 Password 1854 7537 2698 0 0 10235 1854 0.000000 1.000000 0.846637 0.000000 -Salt 45 76 2 0 0 78 45 0.000000 1.000000 0.634146 0.000000 +Salt 45 77 2 0 0 79 45 0.000000 1.000000 0.637097 0.000000 Secret 1300 1576 798 0 0 2374 1300 0.000000 1.000000 0.646162 0.000000 Seed 1 6 0 0 0 6 1 0.000000 1.000000 0.857143 0.000000 Slack Token 4 1 0 0 0 1 4 0.000000 1.000000 0.200000 0.000000 @@ -271,4 +271,4 @@ Token 651 4172 438 Twilio API Key 0 5 2 0 0 7 0 0.000000 1.000000 URL Credentials 210 151 220 0 0 371 210 0.000000 1.000000 0.638554 0.000000 UUID 1069 265 0 0 0 265 1069 0.000000 1.000000 0.198651 0.000000 - 12202 50511 5098 0 0 0 50511 12202 0.000000 1.000000 0.805431 0.000000 + 12203 50512 5098 0 0 0 50512 12203 0.000000 1.000000 0.805421 0.000000 From 84d66a47d07a30bab8e4316b109024cebb042a79 Mon Sep 17 00:00:00 2001 From: Roman Babenko Date: Fri, 6 Sep 2024 14:44:36 +0300 Subject: [PATCH 23/49] Rollback wrong markup --- meta/c2d95db7.csv | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/meta/c2d95db7.csv b/meta/c2d95db7.csv index d7390ab3..bd55addc 100644 --- a/meta/c2d95db7.csv +++ b/meta/c2d95db7.csv @@ -38,7 +38,7 @@ Id,FileID,Domain,RepoName,FilePath,LineStart,LineEnd,GroundTruth,WithWords,Value 29384,a3046da0,GitHub,c2d95db7,data/c2d95db7/test/a3046da0.example,638,638,Template,T,14,27,F,F,Any,,,Unknown,2.61,13,F,F,F,Key 29396,be5fd676,GitHub,c2d95db7,data/c2d95db7/src/be5fd676.conf,428,428,Template,T,33,39,F,F,CharsOnly,,,Secret,2.25,6,F,F,F,Auth:Password 29769,be5fd676,GitHub,c2d95db7,data/c2d95db7/src/be5fd676.conf,55,55,T,T,18,47,F,F,Any,,,Secret,3.67,29,F,F,F,Password:Secret -29785,8e165a9d,GitHub,c2d95db7,data/c2d95db7/src/8e165a9d.sh,371,371,T,F,18,38,F,F,,,,,0.0,0,F,F,F,Password +29785,8e165a9d,GitHub,c2d95db7,data/c2d95db7/src/8e165a9d.sh,371,371,F,F,18,38,F,F,,,,bash_string_manipulation,0.0,0,F,F,F,Password 30241,a3046da0,GitHub,c2d95db7,data/c2d95db7/test/a3046da0.example,802,802,Template,T,29,57,F,F,Any,,,Unknown,3.15,28,F,F,F,Key 30487,a3046da0,GitHub,c2d95db7,data/c2d95db7/test/a3046da0.example,414,414,Template,T,17,23,F,F,CharsOnly,,,Secret,2.25,6,F,F,F,Password 30491,a3046da0,GitHub,c2d95db7,data/c2d95db7/test/a3046da0.example,814,814,T,T,17,33,F,F,CharsOnly,,,Secret,2.75,16,F,F,F,Key:Secret From 28ca4411a77090e1df8689312bde6a2a057fe7db Mon Sep 17 00:00:00 2001 From: Roman Babenko Date: Fri, 6 Sep 2024 15:17:32 +0300 Subject: [PATCH 24/49] BMciFIX --- .ci/benchmark.txt | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.ci/benchmark.txt b/.ci/benchmark.txt index 4402575a..71aa5d95 100644 --- a/.ci/benchmark.txt +++ b/.ci/benchmark.txt @@ -1,5 +1,5 @@ -META MD5 744a34b5e241b09a9e84966b3e3b9bd0 -DATA MD5 d02f5c7efa56379178bb6e7e19e61ebd +META MD5 fb5283f61b933d0bb576d216a462c217 +DATA MD5 dc44046c38f6265cfb8496e7eb41a841 DATA: 16345596 interested lines. MARKUP: 62813 items FileType FileNumber ValidLines Positives Negatives Templates --------------- ------------ ------------ ----------- ----------- ----------- @@ -179,7 +179,7 @@ FileType FileNumber ValidLines Positives Negatives Templat .scala 40 5071 22 101 .scss 16 8553 32 1 .secrets 1 11 1 -.sh 143 21525 55 479 30 +.sh 143 21525 54 480 30 .slim 1 153 1 2 .smali 1 775 18 .snap 3 1708 9 30 2 @@ -223,7 +223,7 @@ FileType FileNumber ValidLines Positives Negatives Templat .yml 419 36169 555 888 378 .zsh 6 872 12 .zsh-theme 1 97 1 -TOTAL: 10264 16345596 12203 50512 5098 +TOTAL: 10264 16345596 12202 50513 5098 credsweeper result_cnt : 0, lost_cnt : 0, true_cnt : 0, false_cnt : 0 Rules Positives Negatives Templates Reported TP FP TN FN FPR FNR ACC PRC RCL F1 ------------------------------ ----------- ----------- ----------- ---------- ---- ---- ----- ----- -------- -------- -------- ----- -------- ---- @@ -262,7 +262,7 @@ Key 3907 15721 482 Nonce 91 49 0 0 0 49 91 0.000000 1.000000 0.350000 0.000000 Other 0 8291 1 0 0 8292 0 0.000000 1.000000 PEM Private Key 1019 1483 0 0 0 1483 1019 0.000000 1.000000 0.592726 0.000000 -Password 1854 7537 2698 0 0 10235 1854 0.000000 1.000000 0.846637 0.000000 +Password 1853 7538 2698 0 0 10236 1853 0.000000 1.000000 0.846720 0.000000 Salt 45 77 2 0 0 79 45 0.000000 1.000000 0.637097 0.000000 Secret 1300 1576 798 0 0 2374 1300 0.000000 1.000000 0.646162 0.000000 Seed 1 6 0 0 0 6 1 0.000000 1.000000 0.857143 0.000000 @@ -271,4 +271,4 @@ Token 651 4172 438 Twilio API Key 0 5 2 0 0 7 0 0.000000 1.000000 URL Credentials 210 151 220 0 0 371 210 0.000000 1.000000 0.638554 0.000000 UUID 1069 265 0 0 0 265 1069 0.000000 1.000000 0.198651 0.000000 - 12203 50512 5098 0 0 0 50512 12203 0.000000 1.000000 0.805421 0.000000 + 12202 50513 5098 0 0 0 50513 12202 0.000000 1.000000 0.805437 0.000000 From ff1f812303442670543d12d36ca9bbc999d90a01 Mon Sep 17 00:00:00 2001 From: Roman Babenko Date: Fri, 6 Sep 2024 16:31:47 +0300 Subject: [PATCH 25/49] markup --- download_data.py | 2 ++ meta/2ba83c6a.csv | 3 ++- meta/81cd05d0.csv | 2 +- meta/ec138349.csv | 4 ++-- 4 files changed, 7 insertions(+), 4 deletions(-) diff --git a/download_data.py b/download_data.py index df82d594..af8a1a73 100644 --- a/download_data.py +++ b/download_data.py @@ -326,6 +326,8 @@ def get_obfuscated_value(value, meta_row: MetaRow): obfuscated_value = value[:7] + generate_value(value[7:]) elif value.startswith("hexpass:"): obfuscated_value = value[:8] + generate_value(value[8:]) + elif value.startswith("hexsalt:"): + obfuscated_value = value[:8] + generate_value(value[8:]) elif value.startswith("SWMTKN-1-"): obfuscated_value = value[:9] + generate_value(value[9:]) elif value.startswith("hooks.slack.com/services/"): diff --git a/meta/2ba83c6a.csv b/meta/2ba83c6a.csv index 5f942ac1..344d8542 100644 --- a/meta/2ba83c6a.csv +++ b/meta/2ba83c6a.csv @@ -7680,7 +7680,7 @@ Id,FileID,Domain,RepoName,FilePath,LineStart,LineEnd,GroundTruth,WithWords,Value 111029,7b9b02b8,GitHub,2ba83c6a,data/2ba83c6a/test/7b9b02b8.txt,41,41,F,F,,,F,F,,,,,0.00,,F,F,F,Password 111030,7b9b02b8,GitHub,2ba83c6a,data/2ba83c6a/test/7b9b02b8.txt,26,26,F,F,,,F,F,,,,,0.00,,F,F,F,Salt 111031,7b9b02b8,GitHub,2ba83c6a,data/2ba83c6a/test/7b9b02b8.txt,34,34,F,F,,,F,F,,,,,0.00,,F,F,F,Salt -111032,7b9b02b8,GitHub,2ba83c6a,data/2ba83c6a/test/7b9b02b8.txt,42,42,F,F,,,F,F,,,,,0.00,,F,F,F,Salt +111032,7b9b02b8,GitHub,2ba83c6a,data/2ba83c6a/test/7b9b02b8.txt,42,42,T,F,15,31,F,F,,,,,0.00,,F,F,F,Salt 111033,7b9b02b8,GitHub,2ba83c6a,data/2ba83c6a/test/7b9b02b8.txt,50,50,F,F,,,F,F,,,,,0.00,,F,F,F,Salt 111034,7b9b02b8,GitHub,2ba83c6a,data/2ba83c6a/test/7b9b02b8.txt,59,59,F,F,,,F,F,,,,,0.00,,F,F,F,Salt 111035,82a92fc8,GitHub,2ba83c6a,data/2ba83c6a/test/82a92fc8.txt,19,19,F,F,-1,-1,F,F,,,,,0.0,-1,F,F,F,Key @@ -10033,3 +10033,4 @@ Id,FileID,Domain,RepoName,FilePath,LineStart,LineEnd,GroundTruth,WithWords,Value 1479378,d975bf65,GitHub,2ba83c6a,data/2ba83c6a/other/d975bf65.md,11116,11116,F,F,19,27,F,F,,,,,0.0,0,F,F,F,CMD Password 1479379,32929be3,GitHub,2ba83c6a,data/2ba83c6a/src/32929be3.c,334,334,F,F,38,47,F,F,,,,,0.0,0,F,F,F,CMD Password 1479380,9a497e27,GitHub,2ba83c6a,data/2ba83c6a/src/9a497e27.c,142,142,F,F,36,44,F,F,,,,,0.0,0,F,F,F,CMD Password +1111032,7b9b02b8,GitHub,2ba83c6a,data/2ba83c6a/test/7b9b02b8.txt,42,42,T,F,24,31,F,F,,,,,0.00,,F,F,F,Salt diff --git a/meta/81cd05d0.csv b/meta/81cd05d0.csv index cdd3baa2..2eb835be 100644 --- a/meta/81cd05d0.csv +++ b/meta/81cd05d0.csv @@ -4597,7 +4597,7 @@ Id,FileID,Domain,RepoName,FilePath,LineStart,LineEnd,GroundTruth,WithWords,Value 113835,11b115b9,GitHub,81cd05d0,data/81cd05d0/src/11b115b9.json,19695,19695,F,F,,,F,F,,,,,0.00,,F,F,F,Certificate:Key 113836,11b115b9,GitHub,81cd05d0,data/81cd05d0/src/11b115b9.json,20191,20191,F,F,,,F,F,,,,,0.00,,F,F,F,Certificate:Key 113837,11b115b9,GitHub,81cd05d0,data/81cd05d0/src/11b115b9.json,20251,20251,F,F,,,F,F,,,,,0.00,,F,F,F,Certificate:Key -113838,1e29595e,GitHub,81cd05d0,data/81cd05d0/src/1e29595e.json,37017,37017,F,F,,,F,F,,,,,0.00,,F,F,F,AWS S3 Bucket +113838,1e29595e,GitHub,81cd05d0,data/81cd05d0/src/1e29595e.json,37017,37017,T,F,35,79,F,F,,,,,0.00,,F,F,F,AWS S3 Bucket 113839,c24a3a45,GitHub,81cd05d0,data/81cd05d0/src/c24a3a45.json,62400,62400,F,F,-1,-1,F,F,,,,,0.0,-1,F,F,F,Key 113840,c24a3a45,GitHub,81cd05d0,data/81cd05d0/src/c24a3a45.json,62430,62430,F,F,-1,-1,F,F,,,,,0.0,-1,F,F,F,Key 113841,c24a3a45,GitHub,81cd05d0,data/81cd05d0/src/c24a3a45.json,62556,62556,F,F,-1,-1,F,F,,,,,0.0,-1,F,F,F,Key diff --git a/meta/ec138349.csv b/meta/ec138349.csv index d5375620..6d71f39b 100644 --- a/meta/ec138349.csv +++ b/meta/ec138349.csv @@ -103,7 +103,7 @@ Id,FileID,Domain,RepoName,FilePath,LineStart,LineEnd,GroundTruth,WithWords,Value 46025,944501fe,GitHub,ec138349,data/ec138349/test/944501fe.java,122,122,F,F,-1,-1,F,F,,,,,0.0,0,F,F,F,Other 46029,525afad4,GitHub,ec138349,data/ec138349/test/525afad4.java,255,255,F,F,-1,-1,F,F,,,,,0.0,0,F,F,F,Other 46055,525afad4,GitHub,ec138349,data/ec138349/test/525afad4.java,239,239,F,F,-1,-1,F,F,,,,,0.0,0,F,F,F,Other -46189,2d2abc99,GitHub,ec138349,data/ec138349/test/2d2abc99.java,408,408,F,F,,,F,F,,,,,0,0,F,F,F,Secret +46189,2d2abc99,GitHub,ec138349,data/ec138349/test/2d2abc99.java,408,408,Template,F,62,68,F,F,,,,,0,0,F,F,F,Secret 46227,2f9b15a9,GitHub,ec138349,data/ec138349/test/2f9b15a9.java,153,153,F,F,,,F,F,,,,,0,0,F,F,F,Token 48038,8201325f,GitHub,ec138349,data/ec138349/test/8201325f.yml,8,8,Template,T,14,22,F,F,CharsOnly,,,Unknown,2.75,8,F,F,F,Password 48039,37c07333,GitHub,ec138349,data/ec138349/test/37c07333.yml,8,8,Template,T,14,22,F,F,CharsOnly,,,Unknown,2.75,8,F,F,F,Password @@ -182,5 +182,5 @@ Id,FileID,Domain,RepoName,FilePath,LineStart,LineEnd,GroundTruth,WithWords,Value 1338528,18b43943,GitHub,ec138349,data/ec138349/test/18b43943.java,41,41,F,F,6,101,F,F,,,,,0.0,0,F,F,F,JSON Web Token 1338529,18b43943,GitHub,ec138349,data/ec138349/test/18b43943.java,44,44,F,F,6,101,F,F,,,,,0.0,0,F,F,F,JSON Web Token 1338530,18b43943,GitHub,ec138349,data/ec138349/test/18b43943.java,47,47,F,F,6,101,F,F,,,,,0.0,0,F,F,F,JSON Web Token -1338573,2f9b15a9,GitHub,ec138349,data/ec138349/test/2f9b15a9.java,125,125,F,F,58,65,F,F,,,,,0.0,0,F,F,F,Auth:Token +1338573,2f9b15a9,GitHub,ec138349,data/ec138349/test/2f9b15a9.java,125,125,Template,F,58,65,F,F,,,,,0.0,0,F,F,F,Auth:Token 1338575,2f9b15a9,GitHub,ec138349,data/ec138349/test/2f9b15a9.java,158,158,F,F,58,68,F,F,,,,,0.0,0,F,F,F,Auth:Token From ef35bc709fdab7186625671fc8342e0c6cf12def Mon Sep 17 00:00:00 2001 From: Roman Babenko Date: Fri, 6 Sep 2024 16:33:06 +0300 Subject: [PATCH 26/49] bmci --- .ci/benchmark.txt | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/.ci/benchmark.txt b/.ci/benchmark.txt index 71aa5d95..31794f7c 100644 --- a/.ci/benchmark.txt +++ b/.ci/benchmark.txt @@ -1,5 +1,5 @@ -META MD5 fb5283f61b933d0bb576d216a462c217 -DATA MD5 dc44046c38f6265cfb8496e7eb41a841 +META MD5 d3bd08f52ca9bea45c8692a0aeedb9c8 +DATA MD5 09f6499448aa616b009bc762466386c1 DATA: 16345596 interested lines. MARKUP: 62813 items FileType FileNumber ValidLines Positives Negatives Templates --------------- ------------ ------------ ----------- ----------- ----------- @@ -82,11 +82,11 @@ FileType FileNumber ValidLines Positives Negatives Templat .ipynb 1 134 5 .j 1 241 4 .j2 30 5530 6 186 10 -.java 621 134132 361 1366 170 +.java 621 134132 361 1363 173 .jenkinsfile 1 58 2 6 .jinja2 1 64 2 .js 659 536413 536 2496 328 -.json 851 13046493 1074 10910 140 +.json 851 13046493 1075 10909 140 .jsp 13 3202 1 40 .jsx 7 857 19 .jwt 1 1 2 @@ -209,7 +209,7 @@ FileType FileNumber ValidLines Positives Negatives Templat .ts 583 106730 159 1800 201 .tsx 54 7914 1 114 5 .ttar 1 452 1 -.txt 440 78102 5277 6368 49 +.txt 440 78102 5279 6367 49 .utf8 1 77 2 .vsixmanifest 1 36 1 .vsmdi 1 6 2 @@ -223,16 +223,16 @@ FileType FileNumber ValidLines Positives Negatives Templat .yml 419 36169 555 888 378 .zsh 6 872 12 .zsh-theme 1 97 1 -TOTAL: 10264 16345596 12202 50513 5098 +TOTAL: 10264 16345596 12205 50508 5101 credsweeper result_cnt : 0, lost_cnt : 0, true_cnt : 0, false_cnt : 0 Rules Positives Negatives Templates Reported TP FP TN FN FPR FNR ACC PRC RCL F1 ------------------------------ ----------- ----------- ----------- ---------- ---- ---- ----- ----- -------- -------- -------- ----- -------- ---- API 129 3162 189 0 0 3351 129 0.000000 1.000000 0.962931 0.000000 AWS Client ID 168 21 0 0 0 21 168 0.000000 1.000000 0.111111 0.000000 AWS Multi 75 16 0 0 0 16 75 0.000000 1.000000 0.175824 0.000000 -AWS S3 Bucket 66 24 0 0 0 24 66 0.000000 1.000000 0.266667 0.000000 +AWS S3 Bucket 67 23 0 0 0 23 67 0.000000 1.000000 0.255556 0.000000 Atlassian Old PAT token 27 308 3 0 0 311 27 0.000000 1.000000 0.920118 0.000000 -Auth 420 2739 76 0 0 2815 420 0.000000 1.000000 0.870170 0.000000 +Auth 420 2738 77 0 0 2815 420 0.000000 1.000000 0.870170 0.000000 Azure Access Token 19 0 0 0 0 0 19 1.000000 0.000000 0.000000 BASE64 Private Key 7 4 0 0 0 4 7 0.000000 1.000000 0.363636 0.000000 BASE64 encoded PEM Private Key 7 0 0 0 0 0 7 1.000000 0.000000 0.000000 @@ -263,12 +263,12 @@ Nonce 91 49 0 Other 0 8291 1 0 0 8292 0 0.000000 1.000000 PEM Private Key 1019 1483 0 0 0 1483 1019 0.000000 1.000000 0.592726 0.000000 Password 1853 7538 2698 0 0 10236 1853 0.000000 1.000000 0.846720 0.000000 -Salt 45 77 2 0 0 79 45 0.000000 1.000000 0.637097 0.000000 -Secret 1300 1576 798 0 0 2374 1300 0.000000 1.000000 0.646162 0.000000 +Salt 47 76 2 0 0 78 47 0.000000 1.000000 0.624000 0.000000 +Secret 1300 1575 799 0 0 2374 1300 0.000000 1.000000 0.646162 0.000000 Seed 1 6 0 0 0 6 1 0.000000 1.000000 0.857143 0.000000 Slack Token 4 1 0 0 0 1 4 0.000000 1.000000 0.200000 0.000000 -Token 651 4172 438 0 0 4610 651 0.000000 1.000000 0.876259 0.000000 +Token 651 4171 439 0 0 4610 651 0.000000 1.000000 0.876259 0.000000 Twilio API Key 0 5 2 0 0 7 0 0.000000 1.000000 URL Credentials 210 151 220 0 0 371 210 0.000000 1.000000 0.638554 0.000000 UUID 1069 265 0 0 0 265 1069 0.000000 1.000000 0.198651 0.000000 - 12202 50513 5098 0 0 0 50513 12202 0.000000 1.000000 0.805437 0.000000 + 12205 50508 5101 0 0 0 50508 12205 0.000000 1.000000 0.805383 0.000000 From fbd7c9e22a7f112bfc3ff870905b2f7ccca439b1 Mon Sep 17 00:00:00 2001 From: Roman Babenko Date: Fri, 6 Sep 2024 19:40:25 +0300 Subject: [PATCH 27/49] review markup with cred length --- .ci/benchmark.txt | 46 +++++++++++++++++++++++----------------------- meta/0436af4a.csv | 4 ++-- meta/255bae6f.csv | 2 +- meta/28728ab4.csv | 2 +- meta/2ba83c6a.csv | 2 +- meta/41659445.csv | 6 +++--- meta/48fd3902.csv | 4 ++-- meta/49b08818.csv | 2 +- meta/55031e17.csv | 4 ++-- meta/5504e505.csv | 10 +++++----- meta/71455799.csv | 2 +- meta/7738e44d.csv | 2 +- meta/793e12f9.csv | 6 +++--- meta/798d34aa.csv | 6 +++--- meta/850c2319.csv | 2 +- meta/c9b945fa.csv | 2 +- meta/d94d550a.csv | 2 +- meta/df70e863.csv | 12 ++++++------ meta/e0b41e26.csv | 2 +- meta/fdbe07ac.csv | 10 +++++----- review_data.py | 19 ++++++++++++++++--- 21 files changed, 80 insertions(+), 67 deletions(-) diff --git a/.ci/benchmark.txt b/.ci/benchmark.txt index 31794f7c..3bd5385f 100644 --- a/.ci/benchmark.txt +++ b/.ci/benchmark.txt @@ -1,5 +1,5 @@ -META MD5 d3bd08f52ca9bea45c8692a0aeedb9c8 -DATA MD5 09f6499448aa616b009bc762466386c1 +META MD5 04ce5aa7ab32302044dbc1a84eb42534 +DATA MD5 86104e199a279fb42f569d20d373336a DATA: 16345596 interested lines. MARKUP: 62813 items FileType FileNumber ValidLines Positives Negatives Templates --------------- ------------ ------------ ----------- ----------- ----------- @@ -35,7 +35,7 @@ FileType FileNumber ValidLines Positives Negatives Templat .creds 1 10 1 1 .crlf 1 27 1 .crt 2 4979 211 -.cs 268 79532 160 895 94 +.cs 268 79532 159 895 95 .cshtml 5 180 12 .csp 3 379 9 .csproj 1 14 1 @@ -55,7 +55,7 @@ FileType FileNumber ValidLines Positives Negatives Templat .erb 13 323 27 .erl 4 96 7 .ex 25 4968 5 98 5 -.example 17 1838 75 38 53 +.example 17 1838 76 38 52 .exs 24 4842 8 187 4 .ext 5 211 1 4 2 .fsproj 1 75 1 2 @@ -63,7 +63,7 @@ FileType FileNumber ValidLines Positives Negatives Templat .gd 1 37 1 .gml 3 3075 16 .gni 3 5017 19 -.go 1080 566476 695 4123 738 +.go 1080 566476 685 4123 748 .golden 5 1168 1 13 29 .gradle 45 3265 4 90 100 .graphql 7 420 13 @@ -82,10 +82,10 @@ FileType FileNumber ValidLines Positives Negatives Templat .ipynb 1 134 5 .j 1 241 4 .j2 30 5530 6 186 10 -.java 621 134132 361 1363 173 +.java 621 134132 360 1363 174 .jenkinsfile 1 58 2 6 .jinja2 1 64 2 -.js 659 536413 536 2496 328 +.js 659 536413 532 2496 331 .json 851 13046493 1075 10909 140 .jsp 13 3202 1 40 .jsx 7 857 19 @@ -108,12 +108,12 @@ FileType FileNumber ValidLines Positives Negatives Templat .lock 24 160912 142 .log 2 199 38 52 .lua 10 1924 37 3 -.m 16 13358 11 158 3 +.m 16 13358 10 159 3 .manifest 3 102 9 6 .markdown 3 139 3 1 .markerb 3 12 3 .marko 1 21 2 -.md 675 149568 735 2392 623 +.md 675 149568 733 2392 625 .mdx 3 549 7 .mjml 1 18 1 .mjs 22 4424 75 340 @@ -153,13 +153,13 @@ FileType FileNumber ValidLines Positives Negatives Templat .pug 2 193 2 .purs 1 69 4 .pxd 1 150 5 2 -.py 890 291553 684 3301 723 +.py 890 291553 679 3301 728 .pyi 4 1361 9 .pyp 1 167 1 .pyx 2 1094 23 .r 4 62 6 3 1 .rake 2 51 2 -.rb 860 131838 260 3337 612 +.rb 860 131838 254 3340 615 .re 1 31 1 .red 1 159 1 .release 1 13 4 @@ -197,7 +197,7 @@ FileType FileNumber ValidLines Positives Negatives Templat .template 19 1633 4 35 11 .test 2 24 25 4 .testsettings 1 21 1 10 -.tf 21 1377 3 29 2 +.tf 21 1377 2 30 2 .tfstate 4 307 22 11 4 .tfvars 1 31 3 2 .tl 2 2161 161 2 @@ -206,7 +206,7 @@ FileType FileNumber ValidLines Positives Negatives Templat .toml 83 2379 53 105 156 .tpl 1 43 1 .travis 1 34 4 3 1 -.ts 583 106730 159 1800 201 +.ts 583 106730 157 1800 203 .tsx 54 7914 1 114 5 .ttar 1 452 1 .txt 440 78102 5279 6367 49 @@ -223,7 +223,7 @@ FileType FileNumber ValidLines Positives Negatives Templat .yml 419 36169 555 888 378 .zsh 6 872 12 .zsh-theme 1 97 1 -TOTAL: 10264 16345596 12205 50508 5101 +TOTAL: 10264 16345596 12173 50513 5127 credsweeper result_cnt : 0, lost_cnt : 0, true_cnt : 0, false_cnt : 0 Rules Positives Negatives Templates Reported TP FP TN FN FPR FNR ACC PRC RCL F1 ------------------------------ ----------- ----------- ----------- ---------- ---- ---- ----- ----- -------- -------- -------- ----- -------- ---- @@ -232,7 +232,7 @@ AWS Client ID 168 21 0 AWS Multi 75 16 0 0 0 16 75 0.000000 1.000000 0.175824 0.000000 AWS S3 Bucket 67 23 0 0 0 23 67 0.000000 1.000000 0.255556 0.000000 Atlassian Old PAT token 27 308 3 0 0 311 27 0.000000 1.000000 0.920118 0.000000 -Auth 420 2738 77 0 0 2815 420 0.000000 1.000000 0.870170 0.000000 +Auth 414 2739 82 0 0 2821 414 0.000000 1.000000 0.872025 0.000000 Azure Access Token 19 0 0 0 0 0 19 1.000000 0.000000 0.000000 BASE64 Private Key 7 4 0 0 0 4 7 0.000000 1.000000 0.363636 0.000000 BASE64 encoded PEM Private Key 7 0 0 0 0 0 7 1.000000 0.000000 0.000000 @@ -243,7 +243,7 @@ CMD Password 21 128 6 CMD Secret 1 1 0 0 0 1 1 0.000000 1.000000 0.500000 0.000000 CMD Token 6 0 0 0 0 0 6 1.000000 0.000000 0.000000 Certificate 23 471 1 0 0 472 23 0.000000 1.000000 0.953535 0.000000 -Credential 95 420 74 0 0 494 95 0.000000 1.000000 0.838710 0.000000 +Credential 93 420 76 0 0 496 93 0.000000 1.000000 0.842105 0.000000 Docker Swarm Token 2 0 0 0 0 0 2 1.000000 0.000000 0.000000 Dropbox App secret 64 139 1 0 0 140 64 0.000000 1.000000 0.686275 0.000000 Facebook Access Token 0 1 0 0 0 1 0 0.000000 1.000000 @@ -258,17 +258,17 @@ Grafana Provisioned API Key 22 1 0 JSON Web Token 170 61 0 0 0 61 170 0.000000 1.000000 0.264069 0.000000 Jira / Confluence PAT token 0 4 0 0 0 4 0 0.000000 1.000000 Jira 2FA 15 6 1 0 0 7 15 0.000000 1.000000 0.318182 0.000000 -Key 3907 15721 482 0 0 16203 3907 0.000000 1.000000 0.805719 0.000000 +Key 3902 15722 486 0 0 16208 3902 0.000000 1.000000 0.805967 0.000000 Nonce 91 49 0 0 0 49 91 0.000000 1.000000 0.350000 0.000000 -Other 0 8291 1 0 0 8292 0 0.000000 1.000000 +Other 8 8291 1 0 0 8292 8 0.000000 1.000000 0.999036 0.000000 PEM Private Key 1019 1483 0 0 0 1483 1019 0.000000 1.000000 0.592726 0.000000 -Password 1853 7538 2698 0 0 10236 1853 0.000000 1.000000 0.846720 0.000000 +Password 1845 7539 2696 0 0 10235 1845 0.000000 1.000000 0.847268 0.000000 Salt 47 76 2 0 0 78 47 0.000000 1.000000 0.624000 0.000000 -Secret 1300 1575 799 0 0 2374 1300 0.000000 1.000000 0.646162 0.000000 +Secret 1296 1575 803 0 0 2378 1296 0.000000 1.000000 0.647251 0.000000 Seed 1 6 0 0 0 6 1 0.000000 1.000000 0.857143 0.000000 Slack Token 4 1 0 0 0 1 4 0.000000 1.000000 0.200000 0.000000 -Token 651 4171 439 0 0 4610 651 0.000000 1.000000 0.876259 0.000000 +Token 636 4171 454 0 0 4625 636 0.000000 1.000000 0.879110 0.000000 Twilio API Key 0 5 2 0 0 7 0 0.000000 1.000000 -URL Credentials 210 151 220 0 0 371 210 0.000000 1.000000 0.638554 0.000000 +URL Credentials 210 153 218 0 0 371 210 0.000000 1.000000 0.638554 0.000000 UUID 1069 265 0 0 0 265 1069 0.000000 1.000000 0.198651 0.000000 - 12205 50508 5101 0 0 0 50508 12205 0.000000 1.000000 0.805383 0.000000 + 12173 50513 5127 0 0 0 50513 12173 0.000000 1.000000 0.805810 0.000000 diff --git a/meta/0436af4a.csv b/meta/0436af4a.csv index fae1ecf3..a9f4e1c1 100644 --- a/meta/0436af4a.csv +++ b/meta/0436af4a.csv @@ -6,7 +6,7 @@ Id,FileID,Domain,RepoName,FilePath,LineStart,LineEnd,GroundTruth,WithWords,Value 71444,4433e6a4,GitHub,0436af4a,data/0436af4a/test/4433e6a4.cs,210,210,T,F,37,47,F,F,,,,,2.92,10,F,F,F,Auth 71445,17ed1848,GitHub,0436af4a,data/0436af4a/other/17ed1848.md,318,318,T,F,58,66,F,F,,,,,2.75,8,F,F,F,URL Credentials 71446,4663e076,GitHub,0436af4a,data/0436af4a/other/4663e076.md,39,39,T,F,26,36,F,F,,,,,3.12,10,F,F,F,Credential -71447,4663e076,GitHub,0436af4a,data/0436af4a/other/4663e076.md,65,65,T,F,26,31,F,F,,,,,1.92,5,F,F,F,Credential +71447,4663e076,GitHub,0436af4a,data/0436af4a/other/4663e076.md,65,65,Template,F,26,31,F,F,,,,,1.92,5,F,F,F,Credential 71448,a9108d0e,GitHub,0436af4a,data/0436af4a/other/a9108d0e.md,730,730,F,F,37,50,F,F,,,,,2.54,13,F,F,F,Credential 71449,94b788c0,GitHub,0436af4a,data/0436af4a/other/94b788c0.md,77,77,T,F,20,56,F,F,,,,,3.54,36,F,F,F,Auth:UUID 71450,94b788c0,GitHub,0436af4a,data/0436af4a/other/94b788c0.md,78,78,T,F,24,44,F,F,,,,,3.58,20,F,F,F,Secret:Auth @@ -86,7 +86,7 @@ Id,FileID,Domain,RepoName,FilePath,LineStart,LineEnd,GroundTruth,WithWords,Value 71525,c9fde51c,GitHub,0436af4a,data/0436af4a/test/c9fde51c.cs,97,97,T,F,36,43,F,F,,,,,2.81,7,F,F,F,Password 71526,d197c0ec,GitHub,0436af4a,data/0436af4a/test/d197c0ec.cs,131,131,T,F,44,58,F,F,,,,,3.25,14,F,F,F,Password 71527,d197c0ec,GitHub,0436af4a,data/0436af4a/test/d197c0ec.cs,132,132,T,F,44,58,F,F,,,,,3.05,14,F,F,F,Password -71528,d2b2603a,GitHub,0436af4a,data/0436af4a/test/d2b2603a.cs,25,25,T,F,36,43,F,F,,,,,2.13,7,F,F,F,Key +71528,d2b2603a,GitHub,0436af4a,data/0436af4a/test/d2b2603a.cs,25,25,Template,F,36,43,F,F,,,,,2.13,7,F,F,F,Key 71529,d2b2603a,GitHub,0436af4a,data/0436af4a/test/d2b2603a.cs,150,150,T,F,41,49,F,F,,,,,2.75,8,F,F,F,Key 71530,df627f1b,GitHub,0436af4a,data/0436af4a/test/df627f1b.cs,169,169,T,F,38,48,F,F,,,,,3.12,10,F,F,F,Password 71531,df627f1b,GitHub,0436af4a,data/0436af4a/test/df627f1b.cs,170,170,T,F,38,49,F,F,,,,,3.28,11,F,F,F,Password diff --git a/meta/255bae6f.csv b/meta/255bae6f.csv index 5924612c..5a52297a 100644 --- a/meta/255bae6f.csv +++ b/meta/255bae6f.csv @@ -138,7 +138,7 @@ Id,FileID,Domain,RepoName,FilePath,LineStart,LineEnd,GroundTruth,WithWords,Value 32302,7d1318d5,GitHub,255bae6f,data/255bae6f/src/7d1318d5.env,1,1,T,F,17,145,F,F,Any,,,Token,3.86,128,F,F,F,Secret:Token 32490,5b3e5159,GitHub,255bae6f,data/255bae6f/src/5b3e5159.env,3,3,T,T,25,39,F,F,CharsOnly,,,Secret,3.18,14,F,F,F,Password 32491,b15907c6,GitHub,255bae6f,data/255bae6f/src/b15907c6.env,4,4,T,T,25,41,F,F,CharsOnly,,,Secret,3.75,16,F,F,F,Password -32592,93cb0617,GitHub,255bae6f,data/255bae6f/src/93cb0617.rb,8,8,T,F,23,28,F,F,Any,,,Token,2.32,5,F,F,F,Token +32592,93cb0617,GitHub,255bae6f,data/255bae6f/src/93cb0617.rb,8,8,Template,F,23,28,F,F,Any,,,Token,2.32,5,F,F,F,Token 32713,88be4086,GitHub,255bae6f,data/255bae6f/test/88be4086.test,11,11,T,T,19,35,F,F,CharsOnly,,,Auth,3.25,16,F,F,F,Auth:Key 32734,88be4086,GitHub,255bae6f,data/255bae6f/test/88be4086.test,8,8,T,T,18,33,F,F,CharsOnly,,,Auth,3.59,15,F,F,F,Auth:Dropbox App secret:Key 32943,88be4086,GitHub,255bae6f,data/255bae6f/test/88be4086.test,2,2,T,F,18,33,F,F,CharsOnly,,,Auth,3.24,15,F,F,F,Auth:Dropbox App secret:Key diff --git a/meta/28728ab4.csv b/meta/28728ab4.csv index 24cd26b2..de7bbfd0 100644 --- a/meta/28728ab4.csv +++ b/meta/28728ab4.csv @@ -391,7 +391,7 @@ Id,FileID,Domain,RepoName,FilePath,LineStart,LineEnd,GroundTruth,WithWords,Value 31729,3630b78d,GitHub,28728ab4,data/28728ab4/other/3630b78d.md,25,25,T,F,23,56,F,F,Any,,,Secret,3.34,33,F,F,F,Secret 31730,eacfe95b,GitHub,28728ab4,data/28728ab4/src/eacfe95b.toml,9,9,T,F,23,56,F,F,Any,,,Secret,3.34,33,F,F,F,Secret 31731,eacfe95b,GitHub,28728ab4,data/28728ab4/src/eacfe95b.toml,10,10,T,F,17,34,F,F,Any,,,Unknown,3.17,17,F,F,F,Key -31733,e197299b,GitHub,28728ab4,data/28728ab4/other/e197299b.md,25,25,T,F,17,21,F,F,Any,,,Unknown,1.72,4,F,F,F,Key +31733,e197299b,GitHub,28728ab4,data/28728ab4/other/e197299b.md,25,25,Template,F,17,21,F,F,Any,,,Unknown,1.72,4,F,F,F,Key 31734,89bf6b41,GitHub,28728ab4,data/28728ab4/src/89bf6b41.toml,11,11,T,F,20,64,F,F,Any,,,Secret,4.86,44,F,F,F,Secret 31735,e197299b,GitHub,28728ab4,data/28728ab4/other/e197299b.md,27,27,T,F,20,64,F,F,Any,,,Secret,4.86,44,F,F,F,Secret 31736,7655b49a,GitHub,28728ab4,data/28728ab4/src/7655b49a.toml,9,9,T,F,14,54,F,F,Any,,,Key,3.68,40,F,F,F,API:Key diff --git a/meta/2ba83c6a.csv b/meta/2ba83c6a.csv index 344d8542..7bb3b231 100644 --- a/meta/2ba83c6a.csv +++ b/meta/2ba83c6a.csv @@ -10033,4 +10033,4 @@ Id,FileID,Domain,RepoName,FilePath,LineStart,LineEnd,GroundTruth,WithWords,Value 1479378,d975bf65,GitHub,2ba83c6a,data/2ba83c6a/other/d975bf65.md,11116,11116,F,F,19,27,F,F,,,,,0.0,0,F,F,F,CMD Password 1479379,32929be3,GitHub,2ba83c6a,data/2ba83c6a/src/32929be3.c,334,334,F,F,38,47,F,F,,,,,0.0,0,F,F,F,CMD Password 1479380,9a497e27,GitHub,2ba83c6a,data/2ba83c6a/src/9a497e27.c,142,142,F,F,36,44,F,F,,,,,0.0,0,F,F,F,CMD Password -1111032,7b9b02b8,GitHub,2ba83c6a,data/2ba83c6a/test/7b9b02b8.txt,42,42,T,F,24,31,F,F,,,,,0.00,,F,F,F,Salt +1111032,7b9b02b8,GitHub,2ba83c6a,data/2ba83c6a/test/7b9b02b8.txt,42,42,T,F,23,31,F,F,,,,,0.00,,F,F,F,Salt diff --git a/meta/41659445.csv b/meta/41659445.csv index badcb37d..ab0d3db0 100644 --- a/meta/41659445.csv +++ b/meta/41659445.csv @@ -74,7 +74,7 @@ Id,FileID,Domain,RepoName,FilePath,LineStart,LineEnd,GroundTruth,WithWords,Value 34874,faf09b36,GitHub,41659445,data/41659445/test/faf09b36.ts,78,78,F,F,12,20,F,F,,,,,0.0,0,F,F,F,Token 34875,faf09b36,GitHub,41659445,data/41659445/test/faf09b36.ts,174,174,F,F,12,20,F,F,,,,,0.0,0,F,F,F,Token 34876,5398b006,GitHub,41659445,data/41659445/test/5398b006.js,207,207,F,F,14,20,F,F,,,,,0.0,0,F,F,F,Token -34877,5398b006,GitHub,41659445,data/41659445/test/5398b006.js,212,212,T,F,14,20,F,F,,,,,0.0,0,F,F,F,Token +34877,5398b006,GitHub,41659445,data/41659445/test/5398b006.js,212,212,Template,F,14,20,F,F,,,,,0.0,0,F,F,F,Token 34879,5398b006,GitHub,41659445,data/41659445/test/5398b006.js,202,202,F,F,14,20,F,F,,,,,0.0,0,F,F,F,Token 34881,18278ad7,GitHub,41659445,data/41659445/test/18278ad7.ts,39,39,F,F,12,18,F,F,,,,,0.0,0,F,F,F,Token 34882,faf09b36,GitHub,41659445,data/41659445/test/faf09b36.ts,162,162,F,F,12,20,F,F,,,,,0.0,0,F,F,F,Token @@ -118,8 +118,8 @@ Id,FileID,Domain,RepoName,FilePath,LineStart,LineEnd,GroundTruth,WithWords,Value 35350,074dc7a7,GitHub,41659445,data/41659445/test/074dc7a7.ts,29,29,F,F,15,21,F,F,,,,,0,0,F,F,F,Token 37409,5398b006,GitHub,41659445,data/41659445/test/5398b006.js,205,205,T,F,15,47,F,F,Any,,,Secret,4.2,32,F,F,F,Secret 37410,bf2cd4b4,GitHub,41659445,data/41659445/test/bf2cd4b4.ts,15,15,T,F,13,29,F,F,Any,,,Secret,3.0,16,F,F,F,Password:Secret -37420,074dc7a7,GitHub,41659445,data/41659445/test/074dc7a7.ts,68,68,T,F,12,18,F,F,,,,,0.0,0,F,F,F,Token -37421,074dc7a7,GitHub,41659445,data/41659445/test/074dc7a7.ts,82,82,T,F,12,18,F,F,,,,,0.0,0,F,F,F,Token +37420,074dc7a7,GitHub,41659445,data/41659445/test/074dc7a7.ts,68,68,Template,F,12,18,F,F,,,,,0.0,0,F,F,F,Token +37421,074dc7a7,GitHub,41659445,data/41659445/test/074dc7a7.ts,82,82,Template,F,12,18,F,F,,,,,0.0,0,F,F,F,Token 37422,074dc7a7,GitHub,41659445,data/41659445/test/074dc7a7.ts,67,67,T,F,13,45,F,F,Any,,,Secret,4.2,32,F,F,F,Secret 37423,074dc7a7,GitHub,41659445,data/41659445/test/074dc7a7.ts,73,73,T,F,13,45,F,F,Any,,,Secret,4.2,32,F,F,F,Secret 37424,074dc7a7,GitHub,41659445,data/41659445/test/074dc7a7.ts,80,80,T,F,13,45,F,F,Any,,,Secret,3.99,32,F,F,F,Bitbucket Client ID:Bitbucket Client Secret:Secret diff --git a/meta/48fd3902.csv b/meta/48fd3902.csv index a451f651..38ba0281 100644 --- a/meta/48fd3902.csv +++ b/meta/48fd3902.csv @@ -80,7 +80,7 @@ Id,FileID,Domain,RepoName,FilePath,LineStart,LineEnd,GroundTruth,WithWords,Value 86723,ca34f69c,GitHub,48fd3902,data/48fd3902/src/ca34f69c.kt,15,15,F,F,114,118,F,F,,,,,1.50,4,F,F,F,Key 86724,ca34f69c,GitHub,48fd3902,data/48fd3902/src/ca34f69c.kt,113,113,F,F,46,50,F,F,,,,,2.0,4,F,F,F,Credential 86725,cd0a11fd,GitHub,48fd3902,data/48fd3902/src/cd0a11fd.yml,46,46,F,F,17,21,F,F,,,,,2.0,4,F,F,F,Key -86726,cd0a11fd,GitHub,48fd3902,data/48fd3902/src/cd0a11fd.yml,48,48,T,F,19,23,F,F,,,,,2.0,4,F,F,F,Auth +86726,cd0a11fd,GitHub,48fd3902,data/48fd3902/src/cd0a11fd.yml,48,48,Template,F,19,23,F,F,,,,,2.0,4,F,F,F,Auth 86727,cf36af35,GitHub,48fd3902,data/48fd3902/src/cf36af35.kt,12,12,F,F,106,110,F,F,,,,,1.50,4,F,F,F,Key 86728,d10a5de0,GitHub,48fd3902,data/48fd3902/src/d10a5de0.yaml,3,3,T,F,20,28,F,F,,,,,3.00,8,F,F,F,Password:Key 86729,d10a5de0,GitHub,48fd3902,data/48fd3902/src/d10a5de0.yaml,4,4,T,F,22,30,F,F,,,,,3.00,8,F,F,F,Password @@ -161,7 +161,7 @@ Id,FileID,Domain,RepoName,FilePath,LineStart,LineEnd,GroundTruth,WithWords,Value 86804,d7af460d,GitHub,48fd3902,data/48fd3902/test/d7af460d.java,28,28,F,F,32,44,F,F,,,,,2.32,12,F,F,F,Password 86805,d7af460d,GitHub,48fd3902,data/48fd3902/test/d7af460d.java,56,56,F,F,24,37,F,F,,,,,2.9,13,F,F,F,Password 86806,d899777e,GitHub,48fd3902,data/48fd3902/test/d899777e.java,92,92,F,F,100,109,F,F,,,,,2.37,9,F,F,F,Password -86807,de90cc15,GitHub,48fd3902,data/48fd3902/test/de90cc15.java,217,217,T,F,32,37,F,F,,,,,1.92,5,F,F,F,Credential +86807,de90cc15,GitHub,48fd3902,data/48fd3902/test/de90cc15.java,217,217,Template,F,32,37,F,F,,,,,1.92,5,F,F,F,Credential 86808,eaeb2b79,GitHub,48fd3902,data/48fd3902/test/eaeb2b79.java,296,296,F,F,93,106,F,F,,,,,2.9,13,F,F,F,Password 86809,eaeb2b79,GitHub,48fd3902,data/48fd3902/test/eaeb2b79.java,333,333,F,F,82,95,F,F,,,,,2.9,13,F,F,F,Password 86810,eaeb2b79,GitHub,48fd3902,data/48fd3902/test/eaeb2b79.java,761,761,F,F,94,107,F,F,,,,,2.9,13,F,F,F,Password diff --git a/meta/49b08818.csv b/meta/49b08818.csv index 1b36d195..c95e7099 100644 --- a/meta/49b08818.csv +++ b/meta/49b08818.csv @@ -3,7 +3,7 @@ Id,FileID,Domain,RepoName,FilePath,LineStart,LineEnd,GroundTruth,WithWords,Value 1512,d096d42d,GitHub,49b08818,data/49b08818/src/d096d42d.txt,1,1,F,F,,,F,F,,,,,0,0,F,F,F,API 3908,6da61865,GitHub,49b08818,data/49b08818/src/6da61865.txt,2,2,F,F,,,F,F,,,,,0,0,F,F,F,API 4082,6f9552a0,GitHub,49b08818,data/49b08818/src/6f9552a0.js,32,32,F,F,,,F,F,,,,,0,0,F,F,F,Auth -1006440,4b49490b,GitHub,49b08818,data/49b08818/src/4b49490b.m,3,3,T,F,30,35,F,F,,,,,0,0,F,F,F,Auth +1006440,4b49490b,GitHub,49b08818,data/49b08818/src/4b49490b.m,3,3,F,F,30,35,F,F,,,,user,0,0,F,F,F,Auth 6440,4b49490b,GitHub,49b08818,data/49b08818/src/4b49490b.m,3,3,T,F,45,53,F,F,,,,,0,0,F,F,F,Password:Auth 6851,c647c1a0,GitHub,49b08818,data/49b08818/src/c647c1a0.java,17,17,F,F,38,-1,F,F,,,,,0.0,0,F,F,F,Auth 1006851,c647c1a0,GitHub,49b08818,data/49b08818/src/c647c1a0.java,17,17,T,F,53,61,F,F,,,,,0.0,0,F,F,F,Auth:Password diff --git a/meta/55031e17.csv b/meta/55031e17.csv index bac95a52..49a76da9 100644 --- a/meta/55031e17.csv +++ b/meta/55031e17.csv @@ -263,7 +263,7 @@ Id,FileID,Domain,RepoName,FilePath,LineStart,LineEnd,GroundTruth,WithWords,Value 100950,dae658b6,GitHub,55031e17,data/55031e17/src/dae658b6.py,86,86,F,F,,,F,F,,,,,0.00,,F,F,F,Password 100951,3a27de4b,GitHub,55031e17,data/55031e17/test/3a27de4b.py,50,50,F,F,,,F,F,,,,,0.00,,F,F,F,Password 100952,aa2879ba,GitHub,55031e17,data/55031e17/test/aa2879ba.py,86,86,F,F,,,F,F,,,,,0.00,,F,F,F,Password -100953,aa2879ba,GitHub,55031e17,data/55031e17/test/aa2879ba.py,98,98,T,F,53,63,F,F,CharOnly,,,NoCredsFound,3.12,10,F,F,F,Password +100953,aa2879ba,GitHub,55031e17,data/55031e17/test/aa2879ba.py,98,98,T,F,53,63,F,F,CharOnly,,,NoCredsFound,3.12,10,F,F,F,Other 100954,aa2879ba,GitHub,55031e17,data/55031e17/test/aa2879ba.py,103,103,F,F,-1,-1,F,F,,,,,0.0,-1,F,F,F,Other 100955,aa2879ba,GitHub,55031e17,data/55031e17/test/aa2879ba.py,111,111,F,F,,,F,F,,,,,0.00,,F,F,F,Password 100956,b8fa2fad,GitHub,55031e17,data/55031e17/test/b8fa2fad.py,258,258,F,F,,,F,F,,,,,0.00,,F,F,F,Password @@ -287,7 +287,7 @@ Id,FileID,Domain,RepoName,FilePath,LineStart,LineEnd,GroundTruth,WithWords,Value 130381,dae658b6,GitHub,55031e17,data/55031e17/src/dae658b6.py,347,347,F,F,,,F,F,,,,,0.00,,F,F,F,Password 130382,4702a6b2,GitHub,55031e17,data/55031e17/src/4702a6b2.py,72,72,F,F,,,F,F,,,,,0.00,,F,F,F,Password 133365,6cb899b4,GitHub,55031e17,data/55031e17/src/6cb899b4.py,178,178,F,F,-1,-1,F,F,,,,,0.0,0,F,F,F,Key -137941,aa2879ba,GitHub,55031e17,data/55031e17/test/aa2879ba.py,98,98,T,F,70,79,F,F,,,,NoCredsFound,0.0,0,F,F,F,Password +137941,aa2879ba,GitHub,55031e17,data/55031e17/test/aa2879ba.py,98,98,T,F,70,79,F,F,,,,NoCredsFound,0.0,0,F,F,F,Other 137942,b4e617dc,GitHub,55031e17,data/55031e17/test/b4e617dc.py,431,431,F,F,51,57,F,F,,,,,0.0,0,F,F,F,Password 1119364,cb11ba4c,GitHub,55031e17,data/55031e17/test/cb11ba4c.py,107,107,T,F,47,68,F,F,,,,,0.0,0,F,F,F,URL Credentials 1137942,b4e617dc,GitHub,55031e17,data/55031e17/test/b4e617dc.py,473,473,T,F,56,72,F,F,,,,,0.0,0,F,F,F,Auth diff --git a/meta/5504e505.csv b/meta/5504e505.csv index 24ac0601..377dc074 100644 --- a/meta/5504e505.csv +++ b/meta/5504e505.csv @@ -15,15 +15,15 @@ Id,FileID,Domain,RepoName,FilePath,LineStart,LineEnd,GroundTruth,WithWords,Value 32435,c32456d4,GitHub,5504e505,data/5504e505/src/c32456d4.rb,34,34,T,F,28,34,F,F,Any,,,Secret,2.58,6,F,F,F,Secret 32437,4b4c548d,GitHub,5504e505,data/5504e505/other/4b4c548d,13,13,T,F,23,36,F,F,Any,,,Secret,2.45,13,F,F,F,Secret 34909,4b4c548d,GitHub,5504e505,data/5504e505/other/4b4c548d,11,11,T,F,13,35,F,F,Any,,,Token,3.55,22,F,F,F,Token -34925,c32456d4,GitHub,5504e505,data/5504e505/src/c32456d4.rb,35,35,T,F,18,24,F,F,Any,,,Token,2.58,6,F,F,F,Token -35222,c32456d4,GitHub,5504e505,data/5504e505/src/c32456d4.rb,36,36,T,F,19,25,F,F,Any,,,Secret,2.58,6,F,F,F,Secret +34925,c32456d4,GitHub,5504e505,data/5504e505/src/c32456d4.rb,35,35,Template,F,18,24,F,F,Any,,,Token,2.58,6,F,F,F,Token +35222,c32456d4,GitHub,5504e505,data/5504e505/src/c32456d4.rb,36,36,Template,F,19,25,F,F,Any,,,Secret,2.58,6,F,F,F,Secret 35227,4b4c548d,GitHub,5504e505,data/5504e505/other/4b4c548d,10,10,T,F,14,26,F,F,CharsOnly,,,Secret,3.58,12,F,F,F,Secret 36698,c32456d4,GitHub,5504e505,data/5504e505/src/c32456d4.rb,48,48,T,F,28,34,F,F,Any,,,Secret,2.58,6,F,F,F,Secret 36699,c32456d4,GitHub,5504e505,data/5504e505/src/c32456d4.rb,62,62,T,F,28,34,F,F,Any,,,Secret,2.58,6,F,F,F,Secret 36700,c32456d4,GitHub,5504e505,data/5504e505/src/c32456d4.rb,76,76,T,F,28,34,F,F,Any,,,Secret,2.58,6,F,F,F,Secret -37282,c32456d4,GitHub,5504e505,data/5504e505/src/c32456d4.rb,49,49,T,F,18,24,F,F,Any,,,Token,2.58,6,F,F,F,Token -37283,c32456d4,GitHub,5504e505,data/5504e505/src/c32456d4.rb,63,63,T,F,18,24,F,F,Any,,,Token,2.58,6,F,F,F,Token -37284,c32456d4,GitHub,5504e505,data/5504e505/src/c32456d4.rb,77,77,T,F,18,24,F,F,Any,,,Token,2.58,6,F,F,F,Token +37282,c32456d4,GitHub,5504e505,data/5504e505/src/c32456d4.rb,49,49,Template,F,18,24,F,F,Any,,,Token,2.58,6,F,F,F,Token +37283,c32456d4,GitHub,5504e505,data/5504e505/src/c32456d4.rb,63,63,Template,F,18,24,F,F,Any,,,Token,2.58,6,F,F,F,Token +37284,c32456d4,GitHub,5504e505,data/5504e505/src/c32456d4.rb,77,77,Template,F,18,24,F,F,Any,,,Token,2.58,6,F,F,F,Token 37411,c32456d4,GitHub,5504e505,data/5504e505/src/c32456d4.rb,50,50,T,F,19,25,F,F,Any,,,Secret,2.58,6,F,F,F,Secret 37412,c32456d4,GitHub,5504e505,data/5504e505/src/c32456d4.rb,64,64,T,F,19,25,F,F,Any,,,Secret,2.58,6,F,F,F,Secret 37413,c32456d4,GitHub,5504e505,data/5504e505/src/c32456d4.rb,78,78,T,F,19,25,F,F,Any,,,Secret,2.58,6,F,F,F,Secret diff --git a/meta/71455799.csv b/meta/71455799.csv index ad10a569..47aa3188 100644 --- a/meta/71455799.csv +++ b/meta/71455799.csv @@ -29,7 +29,7 @@ Id,FileID,Domain,RepoName,FilePath,LineStart,LineEnd,GroundTruth,WithWords,Value 29012,87378121,GitHub,71455799,data/71455799/test/87378121.tfstate,56,56,T,F,25,34,F,F,CharsOnly,,,Secret,2.5,9,F,F,F,Password 29274,f9312afb,GitHub,71455799,data/71455799/test/f9312afb.tf,19,19,T,F,28,37,F,F,CharsOnly,,,Secret,2.5,9,F,F,F,Password 29275,004bbc21,GitHub,71455799,data/71455799/test/004bbc21.tf,19,19,T,F,28,37,F,F,CharsOnly,,,Secret,2.5,9,F,F,F,Password -31371,15c8e8a5,GitHub,71455799,data/71455799/test/15c8e8a5.tf,27,27,T,T,14,30,F,F,CharsOnly,,,Unknown,3.33,16,F,F,F,Key +31371,15c8e8a5,GitHub,71455799,data/71455799/test/15c8e8a5.tf,27,27,F,T,14,30,F,F,CharsOnly,,,Unknown,3.33,16,F,F,F,Key 32234,8d6d06be,GitHub,71455799,data/71455799/test/8d6d06be.go,79,79,T,F,31,51,F,F,Any,,AWS Client ID,Token,3.72,20,F,F,F,AWS Client ID:Gitlab Feed Token:Key 32236,8d6d06be,GitHub,71455799,data/71455799/test/8d6d06be.go,71,71,T,F,31,51,F,F,Any,,AWS Client ID,Token,3.82,20,F,F,F,AWS Client ID:Gitlab Feed Token:Key 32237,8d6d06be,GitHub,71455799,data/71455799/test/8d6d06be.go,75,75,T,F,31,51,F,F,Any,,AWS Client ID,Token,3.72,20,F,F,F,AWS Client ID:Gitlab Feed Token:Key diff --git a/meta/7738e44d.csv b/meta/7738e44d.csv index d000996a..f5bd87a3 100644 --- a/meta/7738e44d.csv +++ b/meta/7738e44d.csv @@ -65,7 +65,7 @@ Id,FileID,Domain,RepoName,FilePath,LineStart,LineEnd,GroundTruth,WithWords,Value 34695,5011ce5c,GitHub,7738e44d,data/7738e44d/src/5011ce5c.yml,45,45,T,F,32,96,F,F,Any,,,Token,3.93,64,F,F,F,Bitbucket Client Secret:Token 34696,0795ee00,GitHub,7738e44d,data/7738e44d/src/0795ee00.yml,45,45,T,F,32,96,F,F,Any,,,Token,3.59,64,F,F,F,Bitbucket Client Secret:Token 34697,e87443ba,GitHub,7738e44d,data/7738e44d/src/e87443ba.yml,45,45,T,F,32,96,F,F,Any,,,Token,3.79,64,F,F,F,Bitbucket Client Secret:Token -35083,7c24a9da,GitHub,7738e44d,data/7738e44d/other/7c24a9da.md,184,184,T,F,20,26,F,F,Any,,,Token,2.58,6,F,F,F,Token +35083,7c24a9da,GitHub,7738e44d,data/7738e44d/other/7c24a9da.md,184,184,Template,F,20,26,F,F,Any,,,Token,2.58,6,F,F,F,Token 40663,c525282e,GitHub,7738e44d,data/7738e44d/src/c525282e.yml,243,243,F,F,,,F,F,,,,,0,0,F,F,F,Bitbucket Client ID 40664,f56f6d9a,GitHub,7738e44d,data/7738e44d/src/f56f6d9a.yml,455,455,F,F,,,F,F,,,,,0,0,F,F,F,Bitbucket Client ID 41756,376478ac,GitHub,7738e44d,data/7738e44d/src/376478ac.yml,11,11,F,F,,,F,F,,,,,0,0,F,F,F,API diff --git a/meta/793e12f9.csv b/meta/793e12f9.csv index 11c86105..7a80b4a4 100644 --- a/meta/793e12f9.csv +++ b/meta/793e12f9.csv @@ -13,14 +13,14 @@ Id,FileID,Domain,RepoName,FilePath,LineStart,LineEnd,GroundTruth,WithWords,Value 28132,b7636b02,GitHub,793e12f9,data/793e12f9/src/b7636b02.rb,126,126,T,T,37,44,F,F,Any,,,Secret,2.81,7,F,F,F,Password 28860,ff081bd4,GitHub,793e12f9,data/793e12f9/src/ff081bd4.rb,453,453,Template,F,48,51,T,F,CharsOnly,,,Token,1.58,3,F,F,F,Token 28888,ff081bd4,GitHub,793e12f9,data/793e12f9/src/ff081bd4.rb,331,331,Template,F,43,49,F,F,CharsOnly,,,Token,2.58,6,F,F,F,Token -30991,9fbcc953,GitHub,793e12f9,data/793e12f9/src/9fbcc953.rb,163,163,F,F,70,73,F,F,CharsOnly,,,SeedSalt,1.58,3,F,F,F,Nonce:Token +30991,9fbcc953,GitHub,793e12f9,data/793e12f9/src/9fbcc953.rb,163,163,F,F,,,F,F,CharsOnly,,,SeedSalt,1.58,3,F,F,F,Nonce:Token 32407,15d532e8,GitHub,793e12f9,data/793e12f9/other/15d532e8.md,106,106,T,F,50,56,T,F,Any,,,Key,2.25,6,F,F,F,API:Key 32573,b89304f6,GitHub,793e12f9,data/793e12f9/src/b89304f6.rb,8,8,F,T,15,28,F,F,CharsOnly,,,Unknown,3.24,13,F,F,F,Key 32578,ee054816,GitHub,793e12f9,data/793e12f9/src/ee054816.html,16,16,T,F,11,43,F,F,Any,,,Key,3.51,32,F,F,F,API:Bitbucket Client ID:Bitbucket Client Secret:Key -34249,89740fcf,GitHub,793e12f9,data/793e12f9/src/89740fcf.rb,38,38,Template,F,53,59,T,F,Any,,,Secret,2.15,6,F,F,T,URL Credentials +34249,89740fcf,GitHub,793e12f9,data/793e12f9/src/89740fcf.rb,38,38,F,F,53,59,T,F,Any,,,Secret,2.15,6,F,F,T,URL Credentials 34458,9fbcc953,GitHub,793e12f9,data/793e12f9/src/9fbcc953.rb,406,406,Template,T,45,54,T,F,Any,,,Secret,2.6,9,F,F,T,Password:URL Credentials 35807,2ade384c,GitHub,793e12f9,data/793e12f9/test/2ade384c,142,142,T,T,49,66,T,F,Any,,,Secret,3.09,17,F,F,T,URL Credentials -36117,89740fcf,GitHub,793e12f9,data/793e12f9/src/89740fcf.rb,6,6,Template,T,56,60,T,F,CharsOnly,,,Secret,1.5,4,F,F,F,Password:URL Credentials +36117,89740fcf,GitHub,793e12f9,data/793e12f9/src/89740fcf.rb,6,6,F,T,56,60,T,F,CharsOnly,,,Secret,1.5,4,F,F,F,Password:URL Credentials 36345,9fbcc953,GitHub,793e12f9,data/793e12f9/src/9fbcc953.rb,154,154,T,T,34,47,T,F,Any,,,Secret,2.9,13,F,F,T,Password:URL Credentials 55574,ff081bd4,GitHub,793e12f9,data/793e12f9/src/ff081bd4.rb,456,456,Template,F,48,51,T,F,CharsOnly,,,Token,1.58,3,F,F,F,Token 100168,89740fcf,GitHub,793e12f9,data/793e12f9/src/89740fcf.rb,8,8,F,F,,,F,F,,,,,0.00,,F,F,F,Password diff --git a/meta/798d34aa.csv b/meta/798d34aa.csv index 4906faf6..15fbdc98 100644 --- a/meta/798d34aa.csv +++ b/meta/798d34aa.csv @@ -60,8 +60,8 @@ Id,FileID,Domain,RepoName,FilePath,LineStart,LineEnd,GroundTruth,WithWords,Value 23689,1c25f72b,GitHub,798d34aa,data/798d34aa/src/1c25f72b.ts,151,151,F,F,,,F,F,,,,,0,0,F,F,F,Token 25102,ad886f81,GitHub,798d34aa,data/798d34aa/src/ad886f81.ts,21,21,F,F,,,F,F,,,,,0,0,F,F,F,Password 27129,3f5c652d,GitHub,798d34aa,data/798d34aa/src/3f5c652d.ts,2,2,F,F,,,F,F,,,,,0,0,F,F,F,Password -27365,9542f82d,GitHub,798d34aa,data/798d34aa/src/9542f82d.yml,223,223,Template,T,34,42,F,F,CharsOnly,,,Secret,3.0,8,F,F,F,Password -27465,74b200a7,GitHub,798d34aa,data/798d34aa/test/74b200a7.example,42,42,Template,T,18,26,F,F,CharsOnly,,,Secret,3.0,8,F,F,F,Password +27365,9542f82d,GitHub,798d34aa,data/798d34aa/src/9542f82d.yml,223,223,T,T,34,42,F,F,CharsOnly,,,Secret,3.0,8,F,F,F,Password +27465,74b200a7,GitHub,798d34aa,data/798d34aa/test/74b200a7.example,42,42,T,T,18,26,F,F,CharsOnly,,,Secret,3.0,8,F,F,F,Password 27705,1a6bfd2b,GitHub,798d34aa,data/798d34aa/src/1a6bfd2b.ts,12,12,Template,T,19,35,F,F,Any,,,Key,3.12,16,F,F,F,API:Key 27763,d4a31c5d,GitHub,798d34aa,data/798d34aa/src/d4a31c5d.ts,19,19,Template,F,13,20,F,F,Any,,,Key,2.41,7,F,F,F,API:Key 27767,9542f82d,GitHub,798d34aa,data/798d34aa/src/9542f82d.yml,23,23,Template,F,34,50,F,F,CharsOnly,,,Secret,3.45,16,F,F,F,Secret @@ -69,7 +69,7 @@ Id,FileID,Domain,RepoName,FilePath,LineStart,LineEnd,GroundTruth,WithWords,Value 28156,74b200a7,GitHub,798d34aa,data/798d34aa/test/74b200a7.example,20,20,T,F,23,38,F,F,CharsOnly,,,Secret,1.0,15,F,F,F,Secret:Dropbox App secret 28157,74b200a7,GitHub,798d34aa,data/798d34aa/test/74b200a7.example,25,25,T,F,23,35,F,F,CharsOnly,,,Secret,1.0,12,F,F,F,Secret 28158,74b200a7,GitHub,798d34aa,data/798d34aa/test/74b200a7.example,15,15,T,F,23,32,F,F,CharsOnly,,,Secret,0.99,9,F,F,F,Secret -28893,b3356305,GitHub,798d34aa,data/798d34aa/other/b3356305.md,207,207,Template,T,19,38,F,F,CharsOnly,,,Secret,3.22,19,F,F,F,Password +28893,b3356305,GitHub,798d34aa,data/798d34aa/other/b3356305.md,207,207,T,T,19,38,F,F,CharsOnly,,,Secret,3.22,19,F,F,F,Password 32212,acf3dd27,GitHub,798d34aa,data/798d34aa/src/acf3dd27.ts,10,10,T,F,21,53,F,F,Any,,,Secret,3.57,32,F,F,F,Bitbucket Client ID:Bitbucket Client Secret:Secret 32426,ddf77893,GitHub,798d34aa,data/798d34aa/src/ddf77893.ts,3,3,T,F,16,74,F,F,Any,,,Secret,3.46,58,F,F,F,Secret 34232,b3356305,GitHub,798d34aa,data/798d34aa/other/b3356305.md,222,222,T,F,26,74,F,F,Any,,,Token,4.0,48,F,F,F,Token diff --git a/meta/850c2319.csv b/meta/850c2319.csv index d83ca6f7..7c472de3 100644 --- a/meta/850c2319.csv +++ b/meta/850c2319.csv @@ -90,7 +90,7 @@ Id,FileID,Domain,RepoName,FilePath,LineStart,LineEnd,GroundTruth,WithWords,Value 101323,e6d458ed,GitHub,850c2319,data/850c2319/test/e6d458ed.go,74,74,Template,T,8,16,F,F,CharOnly,,,Secret,2.75,8,F,F,F,Password 101324,e6d458ed,GitHub,850c2319,data/850c2319/test/e6d458ed.go,75,75,Template,T,8,16,F,F,CharOnly,,,Secret,2.75,8,F,F,F,Password 101325,e6d458ed,GitHub,850c2319,data/850c2319/test/e6d458ed.go,77,77,F,T,,,F,F,CharOnly,,,Secret,2.75,8,F,F,F,Password -101326,999981ff,GitHub,850c2319,data/850c2319/test/999981ff.go,34,34,T,T,17,27,T,F,Any,,,Secret,2.58,16,F,F,F,Password +101326,999981ff,GitHub,850c2319,data/850c2319/test/999981ff.go,34,34,T,T,17,27,T,F,Any,,,Secret,2.58,16,F,F,F,Other 101327,3ea664a1,GitHub,850c2319,data/850c2319/test/3ea664a1.golden,67,67,Template,T,5,13,F,F,CharOnly,,,Secret,2.75,8,F,F,F,Password 101328,3ea664a1,GitHub,850c2319,data/850c2319/test/3ea664a1.golden,551,551,Template,T,3,11,F,F,CharOnly,,,Secret,2.75,8,F,F,F,Password 101329,3ea664a1,GitHub,850c2319,data/850c2319/test/3ea664a1.golden,683,683,Template,T,5,13,F,F,CharOnly,,,Secret,2.75,8,F,F,F,Password diff --git a/meta/c9b945fa.csv b/meta/c9b945fa.csv index 558b4105..67e21416 100644 --- a/meta/c9b945fa.csv +++ b/meta/c9b945fa.csv @@ -80,7 +80,7 @@ Id,FileID,Domain,RepoName,FilePath,LineStart,LineEnd,GroundTruth,WithWords,Value 34858,cb378009,GitHub,c9b945fa,data/c9b945fa/test/cb378009.py,192,192,Template,T,63,78,T,F,Any,,,Token,2.93,15,F,F,F,Token 34859,cb378009,GitHub,c9b945fa,data/c9b945fa/test/cb378009.py,136,136,Template,F,80,85,T,F,CharsOnly,,,Token,2.32,5,F,F,F,Token 34860,cb378009,GitHub,c9b945fa,data/c9b945fa/test/cb378009.py,70,70,Template,F,74,80,T,F,CharsOnly,,,Token,1.92,6,F,F,F,Token -34861,cb378009,GitHub,c9b945fa,data/c9b945fa/test/cb378009.py,74,74,T,T,66,70,T,F,CharsOnly,,,Secret,1.5,4,F,F,F,Password:Secret +34861,cb378009,GitHub,c9b945fa,data/c9b945fa/test/cb378009.py,74,74,Template,T,66,70,T,F,CharsOnly,,,Secret,1.5,4,F,F,F,Password:Secret 34892,0cb23f01,GitHub,c9b945fa,data/c9b945fa/test/0cb23f01.py,157,157,T,F,17,28,F,F,Any,,,Token,3.1,11,F,F,F,Token 34988,64e62af6,GitHub,c9b945fa,data/c9b945fa/src/64e62af6.rst,47,47,T,F,31,71,F,F,Any,,,Token,4.93,40,F,F,F,Auth:Secret:Token 34989,64e62af6,GitHub,c9b945fa,data/c9b945fa/src/64e62af6.rst,106,106,T,F,31,71,F,F,Any,,,Token,4.83,40,F,F,F,Auth:Secret:Token diff --git a/meta/d94d550a.csv b/meta/d94d550a.csv index 7cf09755..fe6e0602 100644 --- a/meta/d94d550a.csv +++ b/meta/d94d550a.csv @@ -97,7 +97,7 @@ Id,FileID,Domain,RepoName,FilePath,LineStart,LineEnd,GroundTruth,WithWords,Value 30872,ad44b60e,GitHub,d94d550a,data/d94d550a/test/ad44b60e.py,29,29,Template,T,21,27,F,F,CharsOnly,,,Token,2.25,6,F,F,F,Token 31321,717f39b8,GitHub,d94d550a,data/d94d550a/test/717f39b8.py,1397,1397,Template,F,54,57,F,F,CharsOnly,,,Secret,1.58,3,F,F,F,Password 31322,717f39b8,GitHub,d94d550a,data/d94d550a/test/717f39b8.py,190,190,Template,F,54,57,F,F,CharsOnly,,,Secret,1.58,3,F,F,F,Password -33789,01c96a2a,GitHub,d94d550a,data/d94d550a/test/01c96a2a.py,1,1,T,F,14,18,F,F,CharsOnly,,,Secret,1.50,4,F,F,F,Password:Key:Secret +33789,01c96a2a,GitHub,d94d550a,data/d94d550a/test/01c96a2a.py,1,1,Template,F,14,18,F,F,CharsOnly,,,Secret,1.50,4,F,F,F,Password:Key:Secret 34217,4b0207e6,GitHub,d94d550a,data/d94d550a/test/4b0207e6.py,36,36,T,F,39,55,F,F,Any,,,Unknown,3.62,16,F,F,F,Key 34219,fc789d4d,GitHub,d94d550a,data/d94d550a/test/fc789d4d.py,52,52,F,F,9,44,F,F,,,,,0.0,0,F,F,F,Other 34285,fc789d4d,GitHub,d94d550a,data/d94d550a/test/fc789d4d.py,29,29,F,F,9,77,F,F,,,,,0.0,0,F,F,F,Other diff --git a/meta/df70e863.csv b/meta/df70e863.csv index b7dac34a..daf7e424 100644 --- a/meta/df70e863.csv +++ b/meta/df70e863.csv @@ -71,15 +71,15 @@ Id,FileID,Domain,RepoName,FilePath,LineStart,LineEnd,GroundTruth,WithWords,Value 32535,7e2d9479,GitHub,df70e863,data/df70e863/test/7e2d9479.go,207,207,Template,T,18,25,F,F,CharsOnly,,,Token,2.81,7,F,F,F,Token 32536,9a5f2fd8,GitHub,df70e863,data/df70e863/src/9a5f2fd8.go,12,12,F,F,28,36,F,F,,,,,0.0,0,F,F,F,Token 32537,7e2d9479,GitHub,df70e863,data/df70e863/test/7e2d9479.go,328,328,Template,T,25,30,F,F,CharsOnly,,,Token,2.32,5,F,F,F,Token -32981,43bcabac,GitHub,df70e863,data/df70e863/test/43bcabac.go,46,46,T,F,15,21,F,F,Any,,,Token,2.58,6,F,F,F,Token -32982,43bcabac,GitHub,df70e863,data/df70e863/test/43bcabac.go,77,77,T,F,15,21,F,F,Any,,,Token,2.58,6,F,F,F,Token +32981,43bcabac,GitHub,df70e863,data/df70e863/test/43bcabac.go,46,46,Template,F,15,21,F,F,Any,,,Token,2.58,6,F,F,F,Token +32982,43bcabac,GitHub,df70e863,data/df70e863/test/43bcabac.go,77,77,Template,F,15,21,F,F,Any,,,Token,2.58,6,F,F,F,Token 34544,1a7980f0,GitHub,df70e863,data/df70e863/test/1a7980f0.go,47,47,T,T,13,19,F,F,CharsOnly,,,Secret,2.25,6,F,F,F,Password 34867,82680089,GitHub,df70e863,data/df70e863/test/82680089.go,41,41,F,F,16,35,F,F,,,,,0.0,0,F,F,F,Key 34897,7e2d9479,GitHub,df70e863,data/df70e863/test/7e2d9479.go,151,151,Template,T,16,24,F,F,CharsOnly,,,Token,2.5,8,F,F,F,Token 34898,7e2d9479,GitHub,df70e863,data/df70e863/test/7e2d9479.go,60,60,Template,T,16,22,F,F,CharsOnly,,,Token,2.25,6,F,F,F,Token 35434,43bcabac,GitHub,df70e863,data/df70e863/test/43bcabac.go,278,278,Template,T,28,42,F,F,Any,,,Token,2.98,14,F,F,F,Token 35435,43bcabac,GitHub,df70e863,data/df70e863/test/43bcabac.go,263,263,Template,T,17,31,F,F,Any,,,Token,2.98,14,F,F,F,Token -36126,43bcabac,GitHub,df70e863,data/df70e863/test/43bcabac.go,387,387,T,F,21,27,F,F,Any,,,Token,2.58,6,F,F,F,Auth:Token +36126,43bcabac,GitHub,df70e863,data/df70e863/test/43bcabac.go,387,387,Template,F,21,27,F,F,Any,,,Token,2.58,6,F,F,F,Auth:Token 36127,384d8831,GitHub,df70e863,data/df70e863/test/384d8831.go,49,49,T,T,23,31,F,F,CharsOnly,,,Token,2.75,8,F,F,F,Auth:Token 36136,327e11c1,GitHub,df70e863,data/df70e863/src/327e11c1.go,22,22,T,F,22,62,F,F,Any,,,Secret,3.78,40,F,F,F,Auth:Secret 36706,7e2d9479,GitHub,df70e863,data/df70e863/test/7e2d9479.go,80,80,Template,T,18,24,F,F,CharsOnly,,,Token,2.25,6,F,F,F,Token @@ -97,9 +97,9 @@ Id,FileID,Domain,RepoName,FilePath,LineStart,LineEnd,GroundTruth,WithWords,Value 37261,7e2d9479,GitHub,df70e863,data/df70e863/test/7e2d9479.go,257,257,Template,T,16,24,F,F,CharsOnly,,,Token,2.5,8,F,F,F,Token 37262,7e2d9479,GitHub,df70e863,data/df70e863/test/7e2d9479.go,72,72,Template,T,16,22,F,F,CharsOnly,,,Token,2.25,6,F,F,F,Token 37263,7e2d9479,GitHub,df70e863,data/df70e863/test/7e2d9479.go,135,135,Template,T,16,22,F,F,CharsOnly,,,Token,2.25,6,F,F,F,Token -37586,43bcabac,GitHub,df70e863,data/df70e863/test/43bcabac.go,413,413,T,F,21,27,F,F,Any,,,Token,2.58,6,F,F,F,Auth:Token -37587,43bcabac,GitHub,df70e863,data/df70e863/test/43bcabac.go,444,444,T,F,21,27,F,F,Any,,,Token,2.58,6,F,F,F,Auth:Token -37588,43bcabac,GitHub,df70e863,data/df70e863/test/43bcabac.go,468,468,T,F,21,27,F,F,Any,,,Token,2.58,6,F,F,F,Auth:Token +37586,43bcabac,GitHub,df70e863,data/df70e863/test/43bcabac.go,413,413,Template,F,21,27,F,F,Any,,,Token,2.58,6,F,F,F,Auth:Token +37587,43bcabac,GitHub,df70e863,data/df70e863/test/43bcabac.go,444,444,Template,F,21,27,F,F,Any,,,Token,2.58,6,F,F,F,Auth:Token +37588,43bcabac,GitHub,df70e863,data/df70e863/test/43bcabac.go,468,468,Template,F,21,27,F,F,Any,,,Token,2.58,6,F,F,F,Auth:Token 37589,384d8831,GitHub,df70e863,data/df70e863/test/384d8831.go,65,65,T,T,23,31,F,F,CharsOnly,,,Token,2.75,8,F,F,F,Auth:Token 37590,384d8831,GitHub,df70e863,data/df70e863/test/384d8831.go,81,81,T,T,23,31,F,F,CharsOnly,,,Token,2.75,8,F,F,F,Auth:Token 37591,384d8831,GitHub,df70e863,data/df70e863/test/384d8831.go,98,98,T,T,23,31,F,F,CharsOnly,,,Token,2.75,8,F,F,F,Auth:Token diff --git a/meta/e0b41e26.csv b/meta/e0b41e26.csv index d8b58d1b..df8c59b8 100644 --- a/meta/e0b41e26.csv +++ b/meta/e0b41e26.csv @@ -45,7 +45,7 @@ Id,FileID,Domain,RepoName,FilePath,LineStart,LineEnd,GroundTruth,WithWords,Value 34587,1efcc29b,GitHub,e0b41e26,data/e0b41e26/src/1efcc29b.js,436,436,T,F,19,35,F,F,Any,,,Secret,3.62,16,F,F,F,Password 35229,9ce19240,GitHub,e0b41e26,data/e0b41e26/src/9ce19240.yaml,82,82,T,F,21,61,F,F,Any,,,Secret,5.0,40,F,F,F,Secret:Key 35231,1efcc29b,GitHub,e0b41e26,data/e0b41e26/src/1efcc29b.js,496,496,T,F,26,66,F,F,Any,,,Secret,5.0,40,F,F,F,Secret:Key -35232,508b8489,GitHub,e0b41e26,data/e0b41e26/test/508b8489.js,1319,1319,T,F,34,38,F,F,CharsOnly,,,Secret,2.0,4,F,F,F,Password:Key:Secret +35232,508b8489,GitHub,e0b41e26,data/e0b41e26/test/508b8489.js,1319,1319,Template,F,34,38,F,F,CharsOnly,,,Secret,2.0,4,F,F,F,Key:Secret 35766,27f723c6,GitHub,e0b41e26,data/e0b41e26/src/27f723c6.js,20,20,T,F,64,94,F,F,Any,,,Token,4.44,30,F,F,F,Auth:Token 35863,c72880b5,GitHub,e0b41e26,data/e0b41e26/test/c72880b5.js,142,142,T,F,28,58,F,F,Any,,,Token,4.71,30,F,F,F,Token 35865,c72880b5,GitHub,e0b41e26,data/e0b41e26/test/c72880b5.js,127,127,T,F,23,43,F,F,Any,,,Key,3.82,20,F,F,F,API:Gitlab Feed Token:Key diff --git a/meta/fdbe07ac.csv b/meta/fdbe07ac.csv index 8793555c..0fade0cc 100644 --- a/meta/fdbe07ac.csv +++ b/meta/fdbe07ac.csv @@ -18,7 +18,7 @@ Id,FileID,Domain,RepoName,FilePath,LineStart,LineEnd,GroundTruth,WithWords,Value 16842,84760b8a,GitHub,fdbe07ac,data/fdbe07ac/test/84760b8a.go,52,52,F,F,,,F,F,,,,,0,0,F,F,F,Password 16843,2b6903fb,GitHub,fdbe07ac,data/fdbe07ac/test/2b6903fb.go,52,52,F,F,,,F,F,,,,,0,0,F,F,F,Password 22531,2b6903fb,GitHub,fdbe07ac,data/fdbe07ac/test/2b6903fb.go,45,45,F,T,7,15,F,F,Any,,,Secret,2.75,8,F,F,F,Password -22557,2b6903fb,GitHub,fdbe07ac,data/fdbe07ac/test/2b6903fb.go,42,42,T,T,7,15,F,F,Any,,,Secret,2.75,8,F,F,F,Password +22557,2b6903fb,GitHub,fdbe07ac,data/fdbe07ac/test/2b6903fb.go,42,42,T,T,7,15,F,F,Any,,,Secret,2.75,8,F,F,F,Other 22558,84760b8a,GitHub,fdbe07ac,data/fdbe07ac/test/84760b8a.go,45,45,F,T,,,F,F,Any,,,Secret,2.75,8,F,F,F,Password 23046,5919a012,GitHub,fdbe07ac,data/fdbe07ac/src/5919a012.go,73,73,F,F,,,F,F,,,,,0,0,F,F,F,Password 24791,932a4f07,GitHub,fdbe07ac,data/fdbe07ac/src/932a4f07.go,702,702,F,F,,,F,F,,,,,0,0,F,F,F,Password @@ -76,7 +76,7 @@ Id,FileID,Domain,RepoName,FilePath,LineStart,LineEnd,GroundTruth,WithWords,Value 42651,4f1e3039,GitHub,fdbe07ac,data/fdbe07ac/src/4f1e3039.yaml,77,77,Template,F,35,41,F,F,Any,,,Secret,2.58,6,F,F,F,Password 46224,35658be4,GitHub,fdbe07ac,data/fdbe07ac/test/35658be4.go,67,67,T,F,14,78,F,F,Any,,,Unknown,3.91,64,F,F,F,Bitbucket Client Secret 46225,35658be4,GitHub,fdbe07ac,data/fdbe07ac/test/35658be4.go,68,68,T,F,15,79,F,F,Any,,,Unknown,3.87,64,F,F,F,Bitbucket Client Secret -46228,84760b8a,GitHub,fdbe07ac,data/fdbe07ac/test/84760b8a.go,42,42,T,T,7,15,F,F,Any,,,Unknown,2.75,8,F,F,F,Password +46228,84760b8a,GitHub,fdbe07ac,data/fdbe07ac/test/84760b8a.go,42,42,T,T,7,15,F,F,Any,,,Unknown,2.75,8,F,F,F,Other 46233,2b6903fb,GitHub,fdbe07ac,data/fdbe07ac/test/2b6903fb.go,37,37,Template,T,32,40,F,F,CharsOnly,,,Secret,2.75,8,F,F,F,Password 46234,84760b8a,GitHub,fdbe07ac,data/fdbe07ac/test/84760b8a.go,37,37,Template,T,32,40,F,F,CharsOnly,,,Secret,2.75,8,F,F,F,Password 46235,2b6903fb,GitHub,fdbe07ac,data/fdbe07ac/test/2b6903fb.go,40,40,Template,T,32,40,F,F,CharsOnly,,,Secret,2.75,8,F,F,F,Password @@ -171,8 +171,8 @@ Id,FileID,Domain,RepoName,FilePath,LineStart,LineEnd,GroundTruth,WithWords,Value 82140,2b6903fb,GitHub,fdbe07ac,data/fdbe07ac/test/2b6903fb.go,61,61,F,F,158,189,F,F,,,,,0.0,0,F,F,F,Password 82141,2b6903fb,GitHub,fdbe07ac,data/fdbe07ac/test/2b6903fb.go,70,70,F,F,176,207,F,F,,,,,0.0,0,F,F,F,Password 82142,2b6903fb,GitHub,fdbe07ac,data/fdbe07ac/test/2b6903fb.go,73,73,F,F,187,218,F,F,,,,,0.0,0,F,F,F,Password -84004,84760b8a,GitHub,fdbe07ac,data/fdbe07ac/test/84760b8a.go,48,48,T,T,7,17,F,F,Any,,,Secret,2.13,10,F,F,F,Password -84005,2b6903fb,GitHub,fdbe07ac,data/fdbe07ac/test/2b6903fb.go,48,48,T,T,7,17,F,F,Any,,,Secret,2.13,10,F,F,F,Password +84004,84760b8a,GitHub,fdbe07ac,data/fdbe07ac/test/84760b8a.go,48,48,T,T,7,17,F,F,Any,,,Secret,2.13,10,F,F,F,Other +84005,2b6903fb,GitHub,fdbe07ac,data/fdbe07ac/test/2b6903fb.go,48,48,T,T,7,17,F,F,Any,,,Secret,2.13,10,F,F,F,Other 100529,0b251a68,GitHub,fdbe07ac,data/fdbe07ac/other/0b251a68.md,407,407,Template,T,3,11,F,F,CharOnly,,,Secret,2.75,8,F,F,F,Password 100530,5919a012,GitHub,fdbe07ac,data/fdbe07ac/src/5919a012.go,171,171,F,F,,,F,F,,,,,0.00,,F,F,F,Password 100531,5919a012,GitHub,fdbe07ac,data/fdbe07ac/src/5919a012.go,295,295,F,F,,,F,F,,,,,0.00,,F,F,F,Password @@ -187,7 +187,7 @@ Id,FileID,Domain,RepoName,FilePath,LineStart,LineEnd,GroundTruth,WithWords,Value 100541,285a035f,GitHub,fdbe07ac,data/fdbe07ac/test/285a035f.go,102,102,T,F,18,35,F,F,Any,,,Key,3.25,17,F,F,F,Key 100542,2ebef03a,GitHub,fdbe07ac,data/fdbe07ac/test/2ebef03a.go,46,46,F,F,,,F,F,,,,,0.00,,F,F,F,Password 100543,2ebef03a,GitHub,fdbe07ac,data/fdbe07ac/test/2ebef03a.go,62,62,F,F,,,F,F,,,,,0.00,,F,F,F,Password -100546,35658be4,GitHub,fdbe07ac,data/fdbe07ac/test/35658be4.go,51,51,T,F,28,44,F,F,Any,,,Secret,3.5,16,F,F,F,Password +100546,35658be4,GitHub,fdbe07ac,data/fdbe07ac/test/35658be4.go,51,51,T,F,3,24,F,F,Any,,,Secret,3.5,16,F,F,F,Other 100547,d8a593d1,GitHub,fdbe07ac,data/fdbe07ac/test/d8a593d1.go,1975,1975,F,F,,,F,F,,,,,0.00,,F,F,F,Password 110132,f15d80cd,GitHub,fdbe07ac,data/fdbe07ac/src/f15d80cd.go,132,132,F,F,,,F,F,,,,,0.00,,F,F,F,Auth 110133,f15d80cd,GitHub,fdbe07ac,data/fdbe07ac/src/f15d80cd.go,31,31,F,F,-1,-1,F,F,,,,,0.0,-1,F,F,F,Other diff --git a/review_data.py b/review_data.py index 34311f8a..af75681a 100644 --- a/review_data.py +++ b/review_data.py @@ -162,9 +162,22 @@ def main(meta_dir: str, if 0 > row.ValueStart: print(f"Missed ValueStart for TRUE markup!\n{row}", flush=True) errors += 1 - elif 0 < row.ValueEnd and 4 > row.ValueEnd - row.ValueStart: - print(f"Too short value for TRUE markup!\n{row}", flush=True) - errors += 1 + if 0 < row.ValueEnd: + categories = row.Category.split(':') + min_length = 6 + if any(x in categories for x in ["Token", "Nonce"]): + # Secrets are like passwords + min_length = 8 + if any(x in categories for x in ["Secret", "CMD ConvertTo-SecureString"]): + # Secrets are like passwords + min_length = 5 + if any(x in categories for x in ["Password", "URL Credentials", "CMD Password"]): + # lost password may be simple but sensitive + min_length = 4 + if min_length > row.ValueEnd - row.ValueStart: + print(f"Too short {min_length} > {row.ValueEnd - row.ValueStart} value for TRUE markup!\n{row}", + flush=True) + errors += 1 elif 0 < row.ValueEnd and "Password" in row.Category and 31 < row.ValueEnd - row.ValueStart: print(f"Too long for Password TRUE markup!\n{row}", flush=True) errors += 1 From 56846bb8fe9be9f5d3dee8686f738b0001653580 Mon Sep 17 00:00:00 2001 From: Roman Babenko Date: Fri, 6 Sep 2024 22:46:34 +0300 Subject: [PATCH 28/49] upd --- meta/2ba83c6a.csv | 1 - meta/372310b3.csv | 1 + meta/6c73b80a.csv | 2 +- meta/81cd05d0.csv | 2 +- meta/a15774b8.csv | 2 +- 5 files changed, 4 insertions(+), 4 deletions(-) diff --git a/meta/2ba83c6a.csv b/meta/2ba83c6a.csv index 7bb3b231..8c69f17d 100644 --- a/meta/2ba83c6a.csv +++ b/meta/2ba83c6a.csv @@ -10033,4 +10033,3 @@ Id,FileID,Domain,RepoName,FilePath,LineStart,LineEnd,GroundTruth,WithWords,Value 1479378,d975bf65,GitHub,2ba83c6a,data/2ba83c6a/other/d975bf65.md,11116,11116,F,F,19,27,F,F,,,,,0.0,0,F,F,F,CMD Password 1479379,32929be3,GitHub,2ba83c6a,data/2ba83c6a/src/32929be3.c,334,334,F,F,38,47,F,F,,,,,0.0,0,F,F,F,CMD Password 1479380,9a497e27,GitHub,2ba83c6a,data/2ba83c6a/src/9a497e27.c,142,142,F,F,36,44,F,F,,,,,0.0,0,F,F,F,CMD Password -1111032,7b9b02b8,GitHub,2ba83c6a,data/2ba83c6a/test/7b9b02b8.txt,42,42,T,F,23,31,F,F,,,,,0.00,,F,F,F,Salt diff --git a/meta/372310b3.csv b/meta/372310b3.csv index 2fb7ed37..2e176f3f 100644 --- a/meta/372310b3.csv +++ b/meta/372310b3.csv @@ -58,6 +58,7 @@ Id,FileID,Domain,RepoName,FilePath,LineStart,LineEnd,GroundTruth,WithWords,Value 41906,975c99ab,GitHub,372310b3,data/372310b3/src/975c99ab.yml,16,16,F,F,,,F,F,,,,,0,0,F,F,F,URL Credentials 41975,da6314b1,GitHub,372310b3,data/372310b3/src/da6314b1.yml,10,10,Template,F,16,22,F,F,NumbersOnly,,,Secret,2.58,6,F,F,F,Password 41976,da6314b1,GitHub,372310b3,data/372310b3/src/da6314b1.yml,23,23,T,F,33,39,T,F,NumbersOnly,,,Secret,2.58,6,F,F,F,URL Credentials +1041976,da6314b1,GitHub,372310b3,data/372310b3/src/da6314b1.yml,23,23,T,F,73,79,T,F,NumbersOnly,,,Secret,2.58,6,F,F,F,URL Credentials 42471,13b30892,GitHub,372310b3,data/372310b3/src/13b30892.yml,12,12,T,F,16,23,F,F,NumbersOnly,,,Secret,2.81,7,F,F,F,Password 42474,9ac63800,GitHub,372310b3,data/372310b3/src/9ac63800.yml,12,12,Template,F,16,22,F,F,NumbersOnly,,,Secret,2.58,6,F,F,F,Password 43105,3362903e,GitHub,372310b3,data/372310b3/src/3362903e.yml,21,21,Template,F,18,24,F,F,NumbersOnly,,,Secret,2.58,6,F,F,F,Password diff --git a/meta/6c73b80a.csv b/meta/6c73b80a.csv index d2a9362d..2a150bea 100644 --- a/meta/6c73b80a.csv +++ b/meta/6c73b80a.csv @@ -320,7 +320,7 @@ Id,FileID,Domain,RepoName,FilePath,LineStart,LineEnd,GroundTruth,WithWords,Value 35293,4c790ff2,GitHub,6c73b80a,data/6c73b80a/test/4c790ff2.key,1,27,T,T,,,F,F,Any,Private,,Unknown,2.53,31,F,F,F,PEM Private Key 1035306,3f79ca86,GitHub,6c73b80a,data/6c73b80a/test/3f79ca86.go,315,315,T,F,18,24,F,F,Any,,,Secret,2.25,6,F,F,F,Password 35306,3f79ca86,GitHub,6c73b80a,data/6c73b80a/test/3f79ca86.go,315,315,T,F,41,47,F,F,Any,,,Secret,2.25,6,F,F,F,Password -35307,3f79ca86,GitHub,6c73b80a,data/6c73b80a/test/3f79ca86.go,330,330,T,F,39,-1,F,F,Any,,,Secret,2.25,6,F,F,F,Password +35307,3f79ca86,GitHub,6c73b80a,data/6c73b80a/test/3f79ca86.go,330,330,T,F,39,45,F,F,Any,,,Secret,2.25,6,F,F,F,Password 35318,8d5e747a,GitHub,6c73b80a,data/6c73b80a/test/8d5e747a.go,275,275,Template,T,31,35,F,F,CharsOnly,,,Secret,1.5,4,F,F,F,Password 35319,598e7639,GitHub,6c73b80a,data/6c73b80a/test/598e7639.go,110,110,Template,T,32,36,F,F,CharsOnly,,,Secret,1.5,4,F,F,F,Password 35321,edc1c568,GitHub,6c73b80a,data/6c73b80a/test/edc1c568.go,1418,1418,Template,T,47,50,F,F,Any,,,Secret,1.58,3,F,F,F,Password diff --git a/meta/81cd05d0.csv b/meta/81cd05d0.csv index 2eb835be..51e4ce45 100644 --- a/meta/81cd05d0.csv +++ b/meta/81cd05d0.csv @@ -609,7 +609,7 @@ Id,FileID,Domain,RepoName,FilePath,LineStart,LineEnd,GroundTruth,WithWords,Value 41476,e6cd866d,GitHub,81cd05d0,data/81cd05d0/src/e6cd866d.json,49200,49200,F,F,-1,-1,F,F,,,,,0.0,0,F,F,F,Other 41477,e6cd866d,GitHub,81cd05d0,data/81cd05d0/src/e6cd866d.json,49201,49201,F,F,,,F,F,,,,,0,0,F,F,F,Bitbucket Client Secret 41478,e6cd866d,GitHub,81cd05d0,data/81cd05d0/src/e6cd866d.json,49217,49217,F,F,,,F,F,,,,,0,0,F,F,F,Bitbucket Client ID -41479,02fd2d8d,GitHub,81cd05d0,data/81cd05d0/src/02fd2d8d.json,916115,916115,F,F,,,F,F,,,,,0,0,F,F,F,Token +41479,02fd2d8d,GitHub,81cd05d0,data/81cd05d0/src/02fd2d8d.json,916115,916115,T,F,25,197,F,F,,,,,0,0,F,F,F,Token 41480,b32761c9,GitHub,81cd05d0,data/81cd05d0/src/b32761c9.json,47273,47273,F,F,-1,-1,F,F,,,,,0.0,0,F,F,F,Other 41488,ad52143f,GitHub,81cd05d0,data/81cd05d0/src/ad52143f.json,9624,9624,F,F,-1,-1,F,F,,,,,0.0,0,F,F,F,Other 41490,76d103a1,GitHub,81cd05d0,data/81cd05d0/src/76d103a1.json,164561,164561,F,F,-1,-1,F,F,,,,,0.0,0,F,F,F,Other diff --git a/meta/a15774b8.csv b/meta/a15774b8.csv index c5212cbd..8b3df0e1 100644 --- a/meta/a15774b8.csv +++ b/meta/a15774b8.csv @@ -280,7 +280,7 @@ Id,FileID,Domain,RepoName,FilePath,LineStart,LineEnd,GroundTruth,WithWords,Value 40572,4edb47b7,GitHub,a15774b8,data/a15774b8/src/4edb47b7.py,201,201,F,F,-1,-1,F,F,,,,,0.0,0,F,F,F,Other 40573,9acf4ac5,GitHub,a15774b8,data/a15774b8/src/9acf4ac5.sh,19,19,F,F,,,F,F,,,,,0,0,F,F,F,Password 40574,98674990,GitHub,a15774b8,data/a15774b8/src/98674990.properties,2,2,T,T,9,17,F,F,Any,,,Secret,2.75,8,F,F,F,Password -40575,7d97eee9,GitHub,a15774b8,data/a15774b8/src/7d97eee9.properties,2,2,Template,T,9,17,F,F,Any,,,Secret,2.75,8,F,F,F,Password +40575,7d97eee9,GitHub,a15774b8,data/a15774b8/src/7d97eee9.properties,2,2,T,T,9,17,F,F,Any,,,Secret,2.75,8,F,F,F,Password 40576,089f248c,GitHub,a15774b8,data/a15774b8/src/089f248c.properties,2,2,T,T,9,17,F,F,Any,,,Secret,2.75,8,F,F,F,Password 40577,089f248c,GitHub,a15774b8,data/a15774b8/src/089f248c.properties,4,4,T,F,8,18,F,F,Any,,,Unknown,2.79,10,F,F,F,Password 40578,4d26ee67,GitHub,a15774b8,data/a15774b8/src/4d26ee67.properties,2,2,T,T,9,20,F,F,Any,,,Secret,3.28,11,F,F,F,Password From 3faac05e7350ce84827849015dc1b470a76174e7 Mon Sep 17 00:00:00 2001 From: Roman Babenko Date: Fri, 6 Sep 2024 22:51:56 +0300 Subject: [PATCH 29/49] CCC --- .ci/benchmark.txt | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/.ci/benchmark.txt b/.ci/benchmark.txt index 3bd5385f..96cc0922 100644 --- a/.ci/benchmark.txt +++ b/.ci/benchmark.txt @@ -1,5 +1,5 @@ -META MD5 04ce5aa7ab32302044dbc1a84eb42534 -DATA MD5 86104e199a279fb42f569d20d373336a +META MD5 1f49b1f70668715d9c772425fa0f5390 +DATA MD5 c4344dfa01a1019c870b329f2020eaec DATA: 16345596 interested lines. MARKUP: 62813 items FileType FileNumber ValidLines Positives Negatives Templates --------------- ------------ ------------ ----------- ----------- ----------- @@ -86,7 +86,7 @@ FileType FileNumber ValidLines Positives Negatives Templat .jenkinsfile 1 58 2 6 .jinja2 1 64 2 .js 659 536413 532 2496 331 -.json 851 13046493 1075 10909 140 +.json 851 13046493 1076 10908 140 .jsp 13 3202 1 40 .jsx 7 857 19 .jwt 1 1 2 @@ -146,7 +146,7 @@ FileType FileNumber ValidLines Positives Negatives Templat .ppk 1 45 36 .private 1 15 1 .proj 1 85 5 -.properties 48 1621 54 29 32 +.properties 48 1621 55 29 31 .proto 5 5768 2 49 .ps1 16 8509 15 74 2 .ps1xml 1 5022 1 @@ -209,7 +209,7 @@ FileType FileNumber ValidLines Positives Negatives Templat .ts 583 106730 157 1800 203 .tsx 54 7914 1 114 5 .ttar 1 452 1 -.txt 440 78102 5279 6367 49 +.txt 440 78102 5278 6367 49 .utf8 1 77 2 .vsixmanifest 1 36 1 .vsmdi 1 6 2 @@ -220,10 +220,10 @@ FileType FileNumber ValidLines Positives Negatives Templat .xml 9 689 9 .xsl 1 311 1 .yaml 137 19004 125 345 42 -.yml 419 36169 555 888 378 +.yml 419 36169 556 888 378 .zsh 6 872 12 .zsh-theme 1 97 1 -TOTAL: 10264 16345596 12173 50513 5127 +TOTAL: 10264 16345596 12175 50512 5126 credsweeper result_cnt : 0, lost_cnt : 0, true_cnt : 0, false_cnt : 0 Rules Positives Negatives Templates Reported TP FP TN FN FPR FNR ACC PRC RCL F1 ------------------------------ ----------- ----------- ----------- ---------- ---- ---- ----- ----- -------- -------- -------- ----- -------- ---- @@ -262,13 +262,13 @@ Key 3902 15722 486 Nonce 91 49 0 0 0 49 91 0.000000 1.000000 0.350000 0.000000 Other 8 8291 1 0 0 8292 8 0.000000 1.000000 0.999036 0.000000 PEM Private Key 1019 1483 0 0 0 1483 1019 0.000000 1.000000 0.592726 0.000000 -Password 1845 7539 2696 0 0 10235 1845 0.000000 1.000000 0.847268 0.000000 -Salt 47 76 2 0 0 78 47 0.000000 1.000000 0.624000 0.000000 +Password 1846 7539 2695 0 0 10234 1846 0.000000 1.000000 0.847185 0.000000 +Salt 46 76 2 0 0 78 46 0.000000 1.000000 0.629032 0.000000 Secret 1296 1575 803 0 0 2378 1296 0.000000 1.000000 0.647251 0.000000 Seed 1 6 0 0 0 6 1 0.000000 1.000000 0.857143 0.000000 Slack Token 4 1 0 0 0 1 4 0.000000 1.000000 0.200000 0.000000 -Token 636 4171 454 0 0 4625 636 0.000000 1.000000 0.879110 0.000000 +Token 637 4170 454 0 0 4624 637 0.000000 1.000000 0.878920 0.000000 Twilio API Key 0 5 2 0 0 7 0 0.000000 1.000000 -URL Credentials 210 153 218 0 0 371 210 0.000000 1.000000 0.638554 0.000000 +URL Credentials 211 153 218 0 0 371 211 0.000000 1.000000 0.637457 0.000000 UUID 1069 265 0 0 0 265 1069 0.000000 1.000000 0.198651 0.000000 - 12173 50513 5127 0 0 0 50513 12173 0.000000 1.000000 0.805810 0.000000 + 12175 50512 5126 0 0 0 50512 12175 0.000000 1.000000 0.805781 0.000000 From c0ac1d8c8d42c47edb4282ebb78974030e0cd78b Mon Sep 17 00:00:00 2001 From: Roman Babenko Date: Fri, 6 Sep 2024 23:38:23 +0300 Subject: [PATCH 30/49] [skip actions] [auxiliary] 2024-09-06T23:38:23+03:00 --- meta/4a099ada.csv | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/meta/4a099ada.csv b/meta/4a099ada.csv index e28a7b1b..c34ca6d2 100644 --- a/meta/4a099ada.csv +++ b/meta/4a099ada.csv @@ -292,3 +292,7 @@ Id,FileID,Domain,RepoName,FilePath,LineStart,LineEnd,GroundTruth,WithWords,Value 1108077,4180450d,GitHub,4a099ada,data/4a099ada/other/4180450d.json,261,261,T,F,25,89,F,F,CharOnly,,,Secret,1.92,6,F,F,F,API:Token 1108087,4180450d,GitHub,4a099ada,data/4a099ada/other/4180450d.json,291,291,T,F,25,89,F,F,CharOnly,,,Secret,1.92,6,F,F,F,API:Token 1108097,4180450d,GitHub,4a099ada,data/4a099ada/other/4180450d.json,321,321,T,F,25,89,F,F,CharOnly,,,Secret,1.92,6,F,F,F,API:Token +1479616,aba179aa,GitHub,4a099ada,data/4a099ada/test/aba179aa.go,140,140,F,F,27,54,F,F,,,,,0.0,0,F,F,F,API:Token +1479618,aba179aa,GitHub,4a099ada,data/4a099ada/test/aba179aa.go,268,268,F,F,27,54,F,F,,,,,0.0,0,F,F,F,API:Token +1479620,aba179aa,GitHub,4a099ada,data/4a099ada/test/aba179aa.go,308,308,F,F,26,53,F,F,,,,,0.0,0,F,F,F,API:Token +1479622,aba179aa,GitHub,4a099ada,data/4a099ada/test/aba179aa.go,416,416,F,F,27,54,F,F,,,,,0.0,0,F,F,F,API:Token From 2ea3935049b372a7f3f03b4ddea439a9203cc21d Mon Sep 17 00:00:00 2001 From: Roman Babenko Date: Fri, 6 Sep 2024 23:40:26 +0300 Subject: [PATCH 31/49] [skip actions] [auxiliary] 2024-09-06T23:40:26+03:00 --- meta/a09d9e50.csv | 1 + meta/f5e5719b.csv | 1 + 2 files changed, 2 insertions(+) diff --git a/meta/a09d9e50.csv b/meta/a09d9e50.csv index e95e4308..27341dde 100644 --- a/meta/a09d9e50.csv +++ b/meta/a09d9e50.csv @@ -408,3 +408,4 @@ Id,FileID,Domain,RepoName,FilePath,LineStart,LineEnd,GroundTruth,WithWords,Value 1479407,054074fe,GitHub,a09d9e50,data/a09d9e50/test/054074fe.ndjson,1187,1187,F,F,1122,1126,F,F,,,,,0.0,0,F,F,F,CMD Password 1479408,054074fe,GitHub,a09d9e50,data/a09d9e50/test/054074fe.ndjson,1404,1404,T,F,431,441,F,F,,,,,0.0,0,F,F,F,CMD ConvertTo-SecureString 1479596,054074fe,GitHub,a09d9e50,data/a09d9e50/test/054074fe.ndjson,367,367,F,F,1851,1867,F,F,,,,,0.0,0,F,F,F,Key:Token +1479623,054074fe,GitHub,a09d9e50,data/a09d9e50/test/054074fe.ndjson,3266,3266,F,F,,,F,F,,,,,0.0,0,F,F,F,Password diff --git a/meta/f5e5719b.csv b/meta/f5e5719b.csv index 8f3d19eb..ef9217ae 100644 --- a/meta/f5e5719b.csv +++ b/meta/f5e5719b.csv @@ -48,3 +48,4 @@ Id,FileID,Domain,RepoName,FilePath,LineStart,LineEnd,GroundTruth,WithWords,Value 113794,3295df72,GitHub,f5e5719b,data/f5e5719b/src/3295df72.sum,168,168,F,F,,,F,F,,,,,0.00,,F,F,F,Auth 113795,3295df72,GitHub,f5e5719b,data/f5e5719b/src/3295df72.sum,169,169,F,F,,,F,F,,,,,0.00,,F,F,F,Auth 113796,3295df72,GitHub,f5e5719b,data/f5e5719b/src/3295df72.sum,170,170,F,F,,,F,F,,,,,0.00,,F,F,F,Auth +1479624,77ef818c,GitHub,f5e5719b,data/f5e5719b/test/77ef818c.go,159,159,T,F,141,175,F,F,,,,,0.0,0,F,F,F,Token From 7b6f0e70ce48ac9431f9221aa9dfd29fb73829c9 Mon Sep 17 00:00:00 2001 From: Roman Babenko Date: Fri, 6 Sep 2024 23:42:01 +0300 Subject: [PATCH 32/49] upd --- .ci/benchmark.txt | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/.ci/benchmark.txt b/.ci/benchmark.txt index 96cc0922..6741d5e8 100644 --- a/.ci/benchmark.txt +++ b/.ci/benchmark.txt @@ -1,6 +1,6 @@ -META MD5 1f49b1f70668715d9c772425fa0f5390 -DATA MD5 c4344dfa01a1019c870b329f2020eaec -DATA: 16345596 interested lines. MARKUP: 62813 items +META MD5 05dd09402a67f9d8c4b247162d61bd96 +DATA MD5 10197484447994f8b00695aaa9cb9bc9 +DATA: 16345596 interested lines. MARKUP: 62819 items FileType FileNumber ValidLines Positives Negatives Templates --------------- ------------ ------------ ----------- ----------- ----------- 194 28318 71 417 90 @@ -63,7 +63,7 @@ FileType FileNumber ValidLines Positives Negatives Templat .gd 1 37 1 .gml 3 3075 16 .gni 3 5017 19 -.go 1080 566476 685 4123 748 +.go 1080 566476 686 4131 748 .golden 5 1168 1 13 29 .gradle 45 3265 4 90 100 .graphql 7 420 13 @@ -125,7 +125,7 @@ FileType FileNumber ValidLines Positives Negatives Templat .mqh 1 1023 2 .msg 1 26644 1 1 .mysql 1 36 2 -.ndjson 2 5006 74 241 2 +.ndjson 2 5006 74 242 2 .nix 4 211 12 .nolint 1 2 1 .odd 1 1281 43 @@ -223,11 +223,11 @@ FileType FileNumber ValidLines Positives Negatives Templat .yml 419 36169 556 888 378 .zsh 6 872 12 .zsh-theme 1 97 1 -TOTAL: 10264 16345596 12175 50512 5126 +TOTAL: 10264 16345596 12176 50521 5126 credsweeper result_cnt : 0, lost_cnt : 0, true_cnt : 0, false_cnt : 0 Rules Positives Negatives Templates Reported TP FP TN FN FPR FNR ACC PRC RCL F1 ------------------------------ ----------- ----------- ----------- ---------- ---- ---- ----- ----- -------- -------- -------- ----- -------- ---- -API 129 3162 189 0 0 3351 129 0.000000 1.000000 0.962931 0.000000 +API 129 3166 189 0 0 3355 129 0.000000 1.000000 0.962974 0.000000 AWS Client ID 168 21 0 0 0 21 168 0.000000 1.000000 0.111111 0.000000 AWS Multi 75 16 0 0 0 16 75 0.000000 1.000000 0.175824 0.000000 AWS S3 Bucket 67 23 0 0 0 23 67 0.000000 1.000000 0.255556 0.000000 @@ -262,13 +262,13 @@ Key 3902 15722 486 Nonce 91 49 0 0 0 49 91 0.000000 1.000000 0.350000 0.000000 Other 8 8291 1 0 0 8292 8 0.000000 1.000000 0.999036 0.000000 PEM Private Key 1019 1483 0 0 0 1483 1019 0.000000 1.000000 0.592726 0.000000 -Password 1846 7539 2695 0 0 10234 1846 0.000000 1.000000 0.847185 0.000000 +Password 1846 7540 2695 0 0 10235 1846 0.000000 1.000000 0.847198 0.000000 Salt 46 76 2 0 0 78 46 0.000000 1.000000 0.629032 0.000000 Secret 1296 1575 803 0 0 2378 1296 0.000000 1.000000 0.647251 0.000000 Seed 1 6 0 0 0 6 1 0.000000 1.000000 0.857143 0.000000 Slack Token 4 1 0 0 0 1 4 0.000000 1.000000 0.200000 0.000000 -Token 637 4170 454 0 0 4624 637 0.000000 1.000000 0.878920 0.000000 +Token 638 4174 454 0 0 4628 638 0.000000 1.000000 0.878845 0.000000 Twilio API Key 0 5 2 0 0 7 0 0.000000 1.000000 URL Credentials 211 153 218 0 0 371 211 0.000000 1.000000 0.637457 0.000000 UUID 1069 265 0 0 0 265 1069 0.000000 1.000000 0.198651 0.000000 - 12175 50512 5126 0 0 0 50512 12175 0.000000 1.000000 0.805781 0.000000 + 12176 50521 5126 0 0 0 50521 12176 0.000000 1.000000 0.805796 0.000000 From d5632b5c34157cd0f7ef276d70e22f59950ccd1b Mon Sep 17 00:00:00 2001 From: Roman Babenko Date: Sat, 7 Sep 2024 00:17:10 +0300 Subject: [PATCH 33/49] fix --- .ci/benchmark.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ci/benchmark.txt b/.ci/benchmark.txt index 6741d5e8..e6e561ea 100644 --- a/.ci/benchmark.txt +++ b/.ci/benchmark.txt @@ -1,4 +1,4 @@ -META MD5 05dd09402a67f9d8c4b247162d61bd96 +META MD5 bef301726f0850666a81474f30cccd81 DATA MD5 10197484447994f8b00695aaa9cb9bc9 DATA: 16345596 interested lines. MARKUP: 62819 items FileType FileNumber ValidLines Positives Negatives Templates From dec40bb79d3a47ebc47263918aea5eb6d9f22610 Mon Sep 17 00:00:00 2001 From: Roman Babenko Date: Sat, 7 Sep 2024 07:44:50 +0300 Subject: [PATCH 34/49] [skip actions] [auxiliary] 2024-09-07T07:44:50+03:00 --- meta/2ba83c6a.csv | 6 +++--- meta/bbb6c1c3.csv | 2 +- meta/bd14e9cf.csv | 4 ++-- meta/bdf8ad40.csv | 8 +++----- meta/be221a57.csv | 8 ++++---- meta/c145598e.csv | 4 ++-- meta/fa71ac83.csv | 2 +- 7 files changed, 16 insertions(+), 18 deletions(-) diff --git a/meta/2ba83c6a.csv b/meta/2ba83c6a.csv index 8c69f17d..c483a76b 100644 --- a/meta/2ba83c6a.csv +++ b/meta/2ba83c6a.csv @@ -963,7 +963,7 @@ Id,FileID,Domain,RepoName,FilePath,LineStart,LineEnd,GroundTruth,WithWords,Value 27645,b40503ed,GitHub,2ba83c6a,data/2ba83c6a/test/b40503ed.txt,167,167,F,F,6,32,F,F,,,,,0,0,F,F,F,Key 27646,b40503ed,GitHub,2ba83c6a,data/2ba83c6a/test/b40503ed.txt,96,96,F,F,6,18,F,F,,,,,0.0,0,F,F,F,Key 27692,3fc35e88,GitHub,2ba83c6a,data/2ba83c6a/test/3fc35e88.t,19,19,F,F,12,28,F,F,Any,,,Unknown,0,16,F,F,F,Password -27693,1e68713b,GitHub,2ba83c6a,data/2ba83c6a/test/1e68713b.t,65,65,Template,T,18,27,F,F,Any,,,Secret,2.07,9,F,F,F,Secret +27693,1e68713b,GitHub,2ba83c6a,data/2ba83c6a/test/1e68713b.t,65,65,T,T,18,27,F,F,Any,,,Secret,2.07,9,F,F,F,Secret 27798,d6f29cb7,GitHub,2ba83c6a,data/2ba83c6a/test/d6f29cb7.cnf,40,40,Template,F,15,20,F,F,NumbersOnly,,,Unknown,2.32,5,F,F,F,Password 27888,1aa4b790,GitHub,2ba83c6a,data/2ba83c6a/other/1aa4b790.in,401,401,Template,T,69,74,F,F,CharsOnly,,,Unknown,1.92,5,F,F,F,Key:Password 27889,a2ceeeea,GitHub,2ba83c6a,data/2ba83c6a/src/a2ceeeea.pl,82,82,F,F,6,10,F,F,,,,,0.0,0,F,F,F,Key @@ -5847,8 +5847,8 @@ Id,FileID,Domain,RepoName,FilePath,LineStart,LineEnd,GroundTruth,WithWords,Value 61137,911dde03,GitHub,2ba83c6a,data/2ba83c6a/test/911dde03.txt,19134,19134,T,F,6,54,F,F,,,,,0.0,0,F,F,F,Key 61138,911dde03,GitHub,2ba83c6a,data/2ba83c6a/test/911dde03.txt,19142,19142,T,F,6,54,F,F,,,,,0.0,0,F,F,F,Key 61139,911dde03,GitHub,2ba83c6a,data/2ba83c6a/test/911dde03.txt,19150,19150,T,F,6,54,F,F,,,,,0.0,0,F,F,F,Key -61140,5e763eae,GitHub,2ba83c6a,data/2ba83c6a/test/5e763eae.txt,237,237,F,F,-1,-1,F,F,,,,,0.0,0,F,F,F,Key:Bitbucket Client Secret:Bitbucket Client ID -61141,08d3f9a3,GitHub,2ba83c6a,data/2ba83c6a/test/08d3f9a3.txt,185,185,F,F,-1,-1,F,F,,,,,0.0,0,F,F,F,Key:Bitbucket Client Secret:Bitbucket Client ID +61140,5e763eae,GitHub,2ba83c6a,data/2ba83c6a/test/5e763eae.txt,237,237,T,F,6,-1,F,F,,,,,0.0,0,F,F,F,Key +61141,08d3f9a3,GitHub,2ba83c6a,data/2ba83c6a/test/08d3f9a3.txt,185,185,T,F,6,-1,F,F,,,,,0.0,0,F,F,F,Key 61142,911dde03,GitHub,2ba83c6a,data/2ba83c6a/test/911dde03.txt,9676,9676,T,F,6,54,F,F,,,,,0.0,0,F,F,F,Key 61143,911dde03,GitHub,2ba83c6a,data/2ba83c6a/test/911dde03.txt,9684,9684,T,F,6,54,F,F,,,,,0.0,0,F,F,F,Key 61144,911dde03,GitHub,2ba83c6a,data/2ba83c6a/test/911dde03.txt,9692,9692,T,F,6,54,F,F,,,,,0.0,0,F,F,F,Key diff --git a/meta/bbb6c1c3.csv b/meta/bbb6c1c3.csv index 0d8f3d24..6e56f7a2 100644 --- a/meta/bbb6c1c3.csv +++ b/meta/bbb6c1c3.csv @@ -78,7 +78,7 @@ Id,FileID,Domain,RepoName,FilePath,LineStart,LineEnd,GroundTruth,WithWords,Value 25862,ead9e51a,GitHub,bbb6c1c3,data/bbb6c1c3/src/ead9e51a.php,24,24,F,F,,,F,F,,,,,0,0,F,F,F,Password 25994,5aac6017,GitHub,bbb6c1c3,data/bbb6c1c3/src/5aac6017.vue,37,37,F,F,,,F,F,,,,,0,0,F,F,F,Password 34269,38aa1eb3,GitHub,bbb6c1c3,data/bbb6c1c3/src/38aa1eb3.php,26,26,T,F,25,57,F,F,Any,,,Key,3.83,32,F,F,F,API:Bitbucket Client ID:Bitbucket Client Secret:Key -36046,af87967d,GitHub,bbb6c1c3,data/bbb6c1c3/test/af87967d.php,40,40,T,T,33,41,T,F,CharsOnly,,,Secret,2.75,8,F,F,F,URL Credentials +36046,af87967d,GitHub,bbb6c1c3,data/bbb6c1c3/test/af87967d.php,40,40,F,T,,,T,F,CharsOnly,,,Secret,2.75,8,F,F,F,URL Credentials:Password 42100,5aac6017,GitHub,bbb6c1c3,data/bbb6c1c3/src/5aac6017.vue,38,38,F,F,,,F,F,,,,,0,0,F,F,F,Password 42478,2f0fcc18,GitHub,bbb6c1c3,data/bbb6c1c3/src/2f0fcc18.php,128,128,F,F,,,F,F,,,,,0,0,F,F,F,Password 42695,96df2d46,GitHub,bbb6c1c3,data/bbb6c1c3/src/96df2d46.php,134,134,F,F,,,F,F,,,,,0,0,F,F,F,Password diff --git a/meta/bd14e9cf.csv b/meta/bd14e9cf.csv index 4d187e0c..54852e44 100644 --- a/meta/bd14e9cf.csv +++ b/meta/bd14e9cf.csv @@ -33,7 +33,7 @@ Id,FileID,Domain,RepoName,FilePath,LineStart,LineEnd,GroundTruth,WithWords,Value 41729,cc99cf26,GitHub,bd14e9cf,data/bd14e9cf/src/cc99cf26.sh,3269,3269,Template,F,11,16,F,F,Any,,,Secret,2.32,5,F,F,F,Password 41913,9653e9d6,GitHub,bd14e9cf,data/bd14e9cf/src/9653e9d6.sh,32,32,F,F,,,F,F,,,,,0,0,F,F,F,Password 42002,b16690b7,GitHub,bd14e9cf,data/bd14e9cf/src/b16690b7.sh,120,120,T,T,36,42,F,F,Any,,,Secret,2.25,6,F,F,F,Password -42195,fb685b23,GitHub,bd14e9cf,data/bd14e9cf/src/fb685b23.sh,44,44,Template,T,103,111,F,F,Any,,,SeedSalt,2.75,8,F,F,F,API:Password:Salt +42195,fb685b23,GitHub,bd14e9cf,data/bd14e9cf/src/fb685b23.sh,44,44,T,T,103,111,F,F,Any,,,SeedSalt,2.75,8,F,F,F,API:Password:Salt 42196,fb685b23,GitHub,bd14e9cf,data/bd14e9cf/src/fb685b23.sh,88,88,T,F,9,49,F,F,Any,,,Token,3.8,40,F,F,F,Token 42197,fb685b23,GitHub,bd14e9cf,data/bd14e9cf/src/fb685b23.sh,93,93,T,F,108,148,F,F,Any,,,Token,3.80,40,F,F,F,Token:Auth 42198,fb685b23,GitHub,bd14e9cf,data/bd14e9cf/src/fb685b23.sh,98,98,T,F,108,148,F,F,Any,,,Token,3.80,40,F,F,F,Token:Auth @@ -46,7 +46,7 @@ Id,FileID,Domain,RepoName,FilePath,LineStart,LineEnd,GroundTruth,WithWords,Value 57202,9653e9d6,GitHub,bd14e9cf,data/bd14e9cf/src/9653e9d6.sh,186,186,F,F,-1,-1,F,F,,,,,0.0,0,F,F,F,Other 63763,cc99cf26,GitHub,bd14e9cf,data/bd14e9cf/src/cc99cf26.sh,1343,1343,F,F,,,F,F,,,,,0,0,F,F,F,Password 64384,cc99cf26,GitHub,bd14e9cf,data/bd14e9cf/src/cc99cf26.sh,1026,1026,F,F,,,F,F,,,,,0,0,F,F,F,Password -74989,9653e9d6,GitHub,bd14e9cf,data/bd14e9cf/src/9653e9d6.sh,27,27,Template,T,28,35,F,F,CharOnly,,,Secret,2.52,7,F,F,F,Password +74989,9653e9d6,GitHub,bd14e9cf,data/bd14e9cf/src/9653e9d6.sh,27,27,T,T,28,35,F,F,CharOnly,,,Secret,2.52,7,F,F,F,Password 100325,b16690b7,GitHub,bd14e9cf,data/bd14e9cf/src/b16690b7.sh,146,146,T,F,79,85,F,F,NumberOnly,,,Secret,0.00,0,F,F,F,Password 130048,cc99cf26,GitHub,bd14e9cf,data/bd14e9cf/src/cc99cf26.sh,639,639,F,F,,,F,F,,,,,0.00,,F,F,F,Password 130049,46f3404e,GitHub,bd14e9cf,data/bd14e9cf/src/46f3404e.bat,11,11,F,F,,,F,F,,,,,0.00,,F,F,F,Password diff --git a/meta/bdf8ad40.csv b/meta/bdf8ad40.csv index 18895a70..ea31a650 100644 --- a/meta/bdf8ad40.csv +++ b/meta/bdf8ad40.csv @@ -46,8 +46,8 @@ Id,FileID,Domain,RepoName,FilePath,LineStart,LineEnd,GroundTruth,WithWords,Value 32598,b6e16108,GitHub,bdf8ad40,data/bdf8ad40/src/b6e16108.go,34,34,F,F,41,50,F,F,,,,,0.0,0,F,F,F,Password 33472,f1a9c0d7,GitHub,bdf8ad40,data/bdf8ad40/src/f1a9c0d7.yml,10,10,T,F,26,30,F,F,CharsOnly,,,Secret,2.0,4,F,F,F,Password 33483,f1a9c0d7,GitHub,bdf8ad40,data/bdf8ad40/src/f1a9c0d7.yml,77,77,T,F,24,36,F,F,Any,,,Secret,3.42,12,F,F,F,Password -33771,e8764f49,GitHub,bdf8ad40,data/bdf8ad40/other/e8764f49.md,53,53,Template,T,11,19,F,F,CharsOnly,,,Secret,1.50,8,F,F,F,Password -33773,2711c6ed,GitHub,bdf8ad40,data/bdf8ad40/src/2711c6ed.ini,32,32,Template,T,11,19,F,F,CharsOnly,,,Secret,1.50,8,F,F,F,Password +33771,e8764f49,GitHub,bdf8ad40,data/bdf8ad40/other/e8764f49.md,53,53,T,T,11,19,F,F,CharsOnly,,,Secret,1.50,8,F,F,F,Password +33773,2711c6ed,GitHub,bdf8ad40,data/bdf8ad40/src/2711c6ed.ini,32,32,T,T,11,19,F,F,CharsOnly,,,Secret,1.50,8,F,F,F,Password 33819,f1a9c0d7,GitHub,bdf8ad40,data/bdf8ad40/src/f1a9c0d7.yml,74,74,T,F,21,33,F,F,Any,,,Secret,3.42,12,F,F,F,Password 35448,5623b191,GitHub,bdf8ad40,data/bdf8ad40/src/5623b191.yml,27,27,T,F,20,32,F,F,Any,,,Secret,3.42,12,F,F,F,Password 35465,5623b191,GitHub,bdf8ad40,data/bdf8ad40/src/5623b191.yml,30,30,T,F,23,35,F,F,Any,,,Secret,3.42,12,F,F,F,Password @@ -59,7 +59,7 @@ Id,FileID,Domain,RepoName,FilePath,LineStart,LineEnd,GroundTruth,WithWords,Value 36997,f1a9c0d7,GitHub,bdf8ad40,data/bdf8ad40/src/f1a9c0d7.yml,38,38,T,F,26,30,F,F,CharsOnly,,,Secret,2.0,4,F,F,F,Password 41616,60e6225b,GitHub,bdf8ad40,data/bdf8ad40/src/60e6225b.go,1631,1631,F,F,-1,-1,F,F,,,,,0.0,0,F,F,F,Key 41617,dcda887e,GitHub,bdf8ad40,data/bdf8ad40/src/dcda887e.go,1631,1631,F,F,-1,-1,F,F,,,,,0.0,0,F,F,F,Key -41618,b6e16108,GitHub,bdf8ad40,data/bdf8ad40/src/b6e16108.go,43,43,Template,T,54,62,F,F,Any,,,Secret,2.75,8,F,F,F,Password +41618,b6e16108,GitHub,bdf8ad40,data/bdf8ad40/src/b6e16108.go,43,43,T,T,64,72,F,F,Any,,,Secret,2.75,8,F,F,F,Password 41619,f1a9c0d7,GitHub,bdf8ad40,data/bdf8ad40/src/f1a9c0d7.yml,44,44,T,F,29,33,F,F,CharsOnly,,,Secret,2.0,4,F,F,F,Password 41622,449d6109,GitHub,bdf8ad40,data/bdf8ad40/src/449d6109.go,1629,1629,F,F,-1,-1,F,F,,,,,0.0,0,F,F,F,Key 41623,a47f50b1,GitHub,bdf8ad40,data/bdf8ad40/src/a47f50b1.go,1629,1629,F,F,-1,-1,F,F,,,,,0.0,0,F,F,F,Key @@ -138,7 +138,6 @@ Id,FileID,Domain,RepoName,FilePath,LineStart,LineEnd,GroundTruth,WithWords,Value 82770,5919a012,GitHub,bdf8ad40,data/bdf8ad40/src/5919a012.go,211,211,F,F,39,43,F,F,,,,,0.0,0,F,F,F,Password 100654,03d780df,GitHub,bdf8ad40,data/bdf8ad40/other/03d780df.md,1425,1425,F,F,-1,-1,F,F,,,,,0.0,-1,F,F,F,Other 100655,03d780df,GitHub,bdf8ad40,data/bdf8ad40/other/03d780df.md,1064,1064,F,F,-1,-1,F,F,,,,,0.0,-1,F,F,F,Other -100656,0b251a68,GitHub,bdf8ad40,data/bdf8ad40/other/0b251a68.md,396,396,Template,T,3,11,F,F,CharOnly,,,Secret,2.75,8,F,F,F,Password 100659,cfa37f2d,GitHub,bdf8ad40,data/bdf8ad40/src/cfa37f2d.go,94,94,F,F,-1,-1,F,F,,,,,0.0,-1,F,F,F,Other 100660,245155dc,GitHub,bdf8ad40,data/bdf8ad40/src/245155dc.go,193,193,F,F,,,F,F,,,,,0.00,,F,F,F,Password 100661,5919a012,GitHub,bdf8ad40,data/bdf8ad40/src/5919a012.go,156,156,F,F,,,F,F,,,,,0.00,,F,F,F,Password @@ -150,7 +149,6 @@ Id,FileID,Domain,RepoName,FilePath,LineStart,LineEnd,GroundTruth,WithWords,Value 110215,ae102102,GitHub,bdf8ad40,data/bdf8ad40/src/ae102102.go,513,513,F,F,,,F,F,,,,,0.00,,F,F,F,Token 110216,245155dc,GitHub,bdf8ad40,data/bdf8ad40/src/245155dc.go,50,50,F,F,-1,-1,F,F,,,,,0.0,-1,F,F,F,Key 110217,26d6e792,GitHub,bdf8ad40,data/bdf8ad40/other/26d6e792.md,125,125,F,F,-1,-1,F,F,,,,,0.0,-1,F,F,F,Key -110218,0b251a68,GitHub,bdf8ad40,data/bdf8ad40/other/0b251a68.md,391,391,F,F,,,F,F,,,,,0.00,,F,F,F,Password 130212,03d780df,GitHub,bdf8ad40,data/bdf8ad40/other/03d780df.md,680,680,F,F,,,F,F,,,,,0.00,,F,F,F,Password 130213,5623b191,GitHub,bdf8ad40,data/bdf8ad40/src/5623b191.yml,12,12,F,F,,,F,F,,,,,0.00,,F,F,F,Password 130214,5919a012,GitHub,bdf8ad40,data/bdf8ad40/src/5919a012.go,397,397,F,F,,,F,F,,,,,0.00,,F,F,F,Password diff --git a/meta/be221a57.csv b/meta/be221a57.csv index 4cbc8e39..60b14b1d 100644 --- a/meta/be221a57.csv +++ b/meta/be221a57.csv @@ -55,10 +55,10 @@ Id,FileID,Domain,RepoName,FilePath,LineStart,LineEnd,GroundTruth,WithWords,Value 102379,5a831ea6,GitHub,be221a57,data/be221a57/other/5a831ea6.md,1974,1974,F,F,,,F,F,,,,,0.00,,F,F,F,Secret 102381,5a831ea6,GitHub,be221a57,data/be221a57/other/5a831ea6.md,1129,1129,F,F,,,F,F,,,,,0.00,,F,F,F,Password 102382,5a831ea6,GitHub,be221a57,data/be221a57/other/5a831ea6.md,1130,1130,F,F,,,F,F,,,,,0.00,,F,F,F,Password -102383,5a831ea6,GitHub,be221a57,data/be221a57/other/5a831ea6.md,1143,1143,Template,T,15,25,F,F,CharOnly,,,Secret,2.45,10,F,F,F,Password -102384,5a831ea6,GitHub,be221a57,data/be221a57/other/5a831ea6.md,1153,1153,F,F,,,F,F,,,,,0.00,,F,F,F,Password -102385,5a831ea6,GitHub,be221a57,data/be221a57/other/5a831ea6.md,1157,1157,F,F,,,F,F,,,,,0.00,,F,F,F,Password -102386,5a831ea6,GitHub,be221a57,data/be221a57/other/5a831ea6.md,1166,1166,Template,T,15,25,F,F,CharOnly,,,Secret,2.45,10,F,F,F,Password +102383,5a831ea6,GitHub,be221a57,data/be221a57/other/5a831ea6.md,1143,1143,T,T,15,25,F,F,CharOnly,,,Secret,2.45,10,F,F,F,Password +102384,5a831ea6,GitHub,be221a57,data/be221a57/other/5a831ea6.md,1153,1153,T,F,17,28,F,F,,,,,0.00,,F,F,F,Password +102385,5a831ea6,GitHub,be221a57,data/be221a57/other/5a831ea6.md,1157,1157,T,F,17,28,F,F,,,,,0.00,,F,F,F,Password +102386,5a831ea6,GitHub,be221a57,data/be221a57/other/5a831ea6.md,1166,1166,T,T,15,25,F,F,CharOnly,,,Secret,2.45,10,F,F,F,Password 102387,2f3e8826,GitHub,be221a57,data/be221a57/src/2f3e8826.ts,13,13,F,F,,,F,F,,,,,0.00,,F,F,F,Gitlab Feed Token 102388,9b99e511,GitHub,be221a57,data/be221a57/test/9b99e511.ts,357,357,Template,T,16,30,F,F,CharOnly,,,Secret,3.09,14,F,F,F,Password 102389,9b99e511,GitHub,be221a57,data/be221a57/test/9b99e511.ts,412,412,Template,T,16,29,F,F,CharOnly,,,Secret,3.09,13,F,F,F,Password diff --git a/meta/c145598e.csv b/meta/c145598e.csv index 3f280538..0da6bc7b 100644 --- a/meta/c145598e.csv +++ b/meta/c145598e.csv @@ -33,8 +33,8 @@ Id,FileID,Domain,RepoName,FilePath,LineStart,LineEnd,GroundTruth,WithWords,Value 26021,150ad80d,GitHub,c145598e,data/c145598e/src/150ad80d.jsp,66,66,F,F,-1,-1,F,F,,,,,0.0,0,F,F,F,Key 26471,a0425b8d,GitHub,c145598e,data/c145598e/src/a0425b8d.jsp,30,30,F,F,,,F,F,,,,,0,0,F,F,F,Password 26499,1aacd97c,GitHub,c145598e,data/c145598e/src/1aacd97c.jsp,135,135,F,F,,,F,F,,,,,0,0,F,F,F,Password -27366,ddcc4e1d,GitHub,c145598e,data/c145598e/src/ddcc4e1d.java,40,40,Template,T,44,52,F,F,CharsOnly,,,Secret,2.75,8,F,F,F,Password -32288,928eada6,GitHub,c145598e,data/c145598e/src/928eada6.properties,9,9,Template,T,21,31,F,F,CharsOnly,,,Secret,3.12,10,F,F,F,Certificate:Key:Password +27366,ddcc4e1d,GitHub,c145598e,data/c145598e/src/ddcc4e1d.java,40,40,T,T,44,52,F,F,CharsOnly,,,Secret,2.75,8,F,F,F,Password +32288,928eada6,GitHub,c145598e,data/c145598e/src/928eada6.properties,9,9,T,T,21,31,F,F,CharsOnly,,,Secret,3.12,10,F,F,F,Certificate:Key:Password 34043,85c03d87,GitHub,c145598e,data/c145598e/src/85c03d87.java,69,69,T,T,30,39,F,F,CharsOnly,,,Secret,3.17,9,F,F,F,Password 42103,010eabfb,GitHub,c145598e,data/c145598e/src/010eabfb.java,1147,1147,F,F,,,F,F,,,,,0,0,F,F,F,Credential 61995,022c0b79,GitHub,c145598e,data/c145598e/src/022c0b79.java,152,152,F,F,-1,-1,F,F,,,,,0.0,0,F,F,F,Key diff --git a/meta/fa71ac83.csv b/meta/fa71ac83.csv index 0ebd752c..e9a22e05 100644 --- a/meta/fa71ac83.csv +++ b/meta/fa71ac83.csv @@ -68,7 +68,7 @@ Id,FileID,Domain,RepoName,FilePath,LineStart,LineEnd,GroundTruth,WithWords,Value 108275,7593a72b,GitHub,fa71ac83,data/fa71ac83/test/7593a72b.js,36,36,F,F,,,F,F,,,,,0.00,,F,F,F,Auth 108276,5222ac44,GitHub,fa71ac83,data/fa71ac83/test/5222ac44.js,114,114,F,F,,,F,F,,,,,0.00,,F,F,F,Password 108277,5222ac44,GitHub,fa71ac83,data/fa71ac83/test/5222ac44.js,128,128,F,F,,,F,F,,,,,0.00,,F,F,F,Password -108278,443510dc,GitHub,fa71ac83,data/fa71ac83/test/443510dc.js,12,12,F,F,,,F,F,,,,,0.00,,F,F,F,Token +108278,443510dc,GitHub,fa71ac83,data/fa71ac83/test/443510dc.js,12,12,F,F,,,F,F,,,,,0.00,,F,F,F,Other 114501,9aa3c9b2,GitHub,fa71ac83,data/fa71ac83/src/9aa3c9b2.js,47,47,F,F,,,F,F,,,,,0.00,,F,F,F,Auth 114502,116fac3d,GitHub,fa71ac83,data/fa71ac83/src/116fac3d.js,5899,5899,F,F,,,F,F,,,,,0.00,,F,F,F,Token 114503,116fac3d,GitHub,fa71ac83,data/fa71ac83/src/116fac3d.js,3534,3534,F,F,-1,-1,F,F,,,,,0.0,-1,F,F,F,Key From 5a55ace1f9d080259dd9192e6da63ffead84702d Mon Sep 17 00:00:00 2001 From: Roman Babenko Date: Sat, 7 Sep 2024 07:50:42 +0300 Subject: [PATCH 35/49] markup --- .ci/benchmark.txt | 50 +++++++++++++++++++++++------------------------ 1 file changed, 25 insertions(+), 25 deletions(-) diff --git a/.ci/benchmark.txt b/.ci/benchmark.txt index e6e561ea..8088d4d0 100644 --- a/.ci/benchmark.txt +++ b/.ci/benchmark.txt @@ -1,6 +1,6 @@ -META MD5 bef301726f0850666a81474f30cccd81 -DATA MD5 10197484447994f8b00695aaa9cb9bc9 -DATA: 16345596 interested lines. MARKUP: 62819 items +META MD5 4c7ae9c79f7c372e2e1a26b41b6f7932 +DATA MD5 d7ff3a303207c700d08ca47cd68a25f1 +DATA: 16345328 interested lines. MARKUP: 62817 items FileType FileNumber ValidLines Positives Negatives Templates --------------- ------------ ------------ ----------- ----------- ----------- 194 28318 71 417 90 @@ -63,7 +63,7 @@ FileType FileNumber ValidLines Positives Negatives Templat .gd 1 37 1 .gml 3 3075 16 .gni 3 5017 19 -.go 1080 566476 686 4131 748 +.go 1080 566476 687 4131 747 .golden 5 1168 1 13 29 .gradle 45 3265 4 90 100 .graphql 7 420 13 @@ -78,11 +78,11 @@ FileType FileNumber ValidLines Positives Negatives Templat .iml 6 699 30 .in 6 2130 6 44 10 .inc 2 56 2 1 -.ini 11 1437 25 12 18 +.ini 11 1437 26 12 17 .ipynb 1 134 5 .j 1 241 4 .j2 30 5530 6 186 10 -.java 621 134132 360 1363 174 +.java 621 134132 361 1363 173 .jenkinsfile 1 58 2 6 .jinja2 1 64 2 .js 659 536413 532 2496 331 @@ -113,7 +113,7 @@ FileType FileNumber ValidLines Positives Negatives Templat .markdown 3 139 3 1 .markerb 3 12 3 .marko 1 21 2 -.md 675 149568 733 2392 625 +.md 674 149300 738 2389 621 .mdx 3 549 7 .mjml 1 18 1 .mjs 22 4424 75 340 @@ -135,7 +135,7 @@ FileType FileNumber ValidLines Positives Negatives Templat .patch 4 109405 4 27 .pbxproj 1 941 2 .pem 48 1169 47 8 -.php 371 75710 129 1620 79 +.php 371 75710 128 1622 79 .pl 16 14727 7 33 .pm 3 744 7 .po 3 2994 15 @@ -146,7 +146,7 @@ FileType FileNumber ValidLines Positives Negatives Templat .ppk 1 45 36 .private 1 15 1 .proj 1 85 5 -.properties 48 1621 55 29 31 +.properties 48 1621 58 29 28 .proto 5 5768 2 49 .ps1 16 8509 15 74 2 .ps1xml 1 5022 1 @@ -179,7 +179,7 @@ FileType FileNumber ValidLines Positives Negatives Templat .scala 40 5071 22 101 .scss 16 8553 32 1 .secrets 1 11 1 -.sh 143 21525 54 480 30 +.sh 143 21525 58 480 26 .slim 1 153 1 2 .smali 1 775 18 .snap 3 1708 9 30 2 @@ -192,7 +192,7 @@ FileType FileNumber ValidLines Positives Negatives Templat .sublime-keymap 1 3 1 .sum 37 22854 283 .svg 1 638 12 -.t 9 1767 24 44 14 +.t 9 1767 25 44 13 .td 2 14002 6 .template 19 1633 4 35 11 .test 2 24 25 4 @@ -209,7 +209,7 @@ FileType FileNumber ValidLines Positives Negatives Templat .ts 583 106730 157 1800 203 .tsx 54 7914 1 114 5 .ttar 1 452 1 -.txt 440 78102 5278 6367 49 +.txt 440 78102 5280 6361 49 .utf8 1 77 2 .vsixmanifest 1 36 1 .vsmdi 1 6 2 @@ -223,11 +223,11 @@ FileType FileNumber ValidLines Positives Negatives Templat .yml 419 36169 556 888 378 .zsh 6 872 12 .zsh-theme 1 97 1 -TOTAL: 10264 16345596 12176 50521 5126 +TOTAL: 10263 16345328 12193 50514 5111 credsweeper result_cnt : 0, lost_cnt : 0, true_cnt : 0, false_cnt : 0 Rules Positives Negatives Templates Reported TP FP TN FN FPR FNR ACC PRC RCL F1 ------------------------------ ----------- ----------- ----------- ---------- ---- ---- ----- ----- -------- -------- -------- ----- -------- ---- -API 129 3166 189 0 0 3355 129 0.000000 1.000000 0.962974 0.000000 +API 130 3166 188 0 0 3354 130 0.000000 1.000000 0.962687 0.000000 AWS Client ID 168 21 0 0 0 21 168 0.000000 1.000000 0.111111 0.000000 AWS Multi 75 16 0 0 0 16 75 0.000000 1.000000 0.175824 0.000000 AWS S3 Bucket 67 23 0 0 0 23 67 0.000000 1.000000 0.255556 0.000000 @@ -236,13 +236,13 @@ Auth 414 2739 82 Azure Access Token 19 0 0 0 0 0 19 1.000000 0.000000 0.000000 BASE64 Private Key 7 4 0 0 0 4 7 0.000000 1.000000 0.363636 0.000000 BASE64 encoded PEM Private Key 7 0 0 0 0 0 7 1.000000 0.000000 0.000000 -Bitbucket Client ID 143 2097 9 0 0 2106 143 0.000000 1.000000 0.936416 0.000000 -Bitbucket Client Secret 301 809 10 0 0 819 301 0.000000 1.000000 0.731250 0.000000 +Bitbucket Client ID 143 2095 9 0 0 2104 143 0.000000 1.000000 0.936360 0.000000 +Bitbucket Client Secret 301 807 10 0 0 817 301 0.000000 1.000000 0.730769 0.000000 CMD ConvertTo-SecureString 13 4 0 0 0 4 13 0.000000 1.000000 0.235294 0.000000 CMD Password 21 128 6 0 0 134 21 0.000000 1.000000 0.864516 0.000000 CMD Secret 1 1 0 0 0 1 1 0.000000 1.000000 0.500000 0.000000 CMD Token 6 0 0 0 0 0 6 1.000000 0.000000 0.000000 -Certificate 23 471 1 0 0 472 23 0.000000 1.000000 0.953535 0.000000 +Certificate 24 471 0 0 0 471 24 0.000000 1.000000 0.951515 0.000000 Credential 93 420 76 0 0 496 93 0.000000 1.000000 0.842105 0.000000 Docker Swarm Token 2 0 0 0 0 0 2 1.000000 0.000000 0.000000 Dropbox App secret 64 139 1 0 0 140 64 0.000000 1.000000 0.686275 0.000000 @@ -258,17 +258,17 @@ Grafana Provisioned API Key 22 1 0 JSON Web Token 170 61 0 0 0 61 170 0.000000 1.000000 0.264069 0.000000 Jira / Confluence PAT token 0 4 0 0 0 4 0 0.000000 1.000000 Jira 2FA 15 6 1 0 0 7 15 0.000000 1.000000 0.318182 0.000000 -Key 3902 15722 486 0 0 16208 3902 0.000000 1.000000 0.805967 0.000000 +Key 3905 15720 485 0 0 16205 3905 0.000000 1.000000 0.805818 0.000000 Nonce 91 49 0 0 0 49 91 0.000000 1.000000 0.350000 0.000000 -Other 8 8291 1 0 0 8292 8 0.000000 1.000000 0.999036 0.000000 +Other 8 8292 1 0 0 8293 8 0.000000 1.000000 0.999036 0.000000 PEM Private Key 1019 1483 0 0 0 1483 1019 0.000000 1.000000 0.592726 0.000000 -Password 1846 7540 2695 0 0 10235 1846 0.000000 1.000000 0.847198 0.000000 -Salt 46 76 2 0 0 78 46 0.000000 1.000000 0.629032 0.000000 -Secret 1296 1575 803 0 0 2378 1296 0.000000 1.000000 0.647251 0.000000 +Password 1857 7538 2685 0 0 10223 1857 0.000000 1.000000 0.846275 0.000000 +Salt 47 76 1 0 0 77 47 0.000000 1.000000 0.620968 0.000000 +Secret 1297 1575 802 0 0 2377 1297 0.000000 1.000000 0.646979 0.000000 Seed 1 6 0 0 0 6 1 0.000000 1.000000 0.857143 0.000000 Slack Token 4 1 0 0 0 1 4 0.000000 1.000000 0.200000 0.000000 -Token 638 4174 454 0 0 4628 638 0.000000 1.000000 0.878845 0.000000 +Token 638 4173 454 0 0 4627 638 0.000000 1.000000 0.878822 0.000000 Twilio API Key 0 5 2 0 0 7 0 0.000000 1.000000 -URL Credentials 211 153 218 0 0 371 211 0.000000 1.000000 0.637457 0.000000 +URL Credentials 210 154 218 0 0 372 210 0.000000 1.000000 0.639175 0.000000 UUID 1069 265 0 0 0 265 1069 0.000000 1.000000 0.198651 0.000000 - 12176 50521 5126 0 0 0 50521 12176 0.000000 1.000000 0.805796 0.000000 + 12193 50514 5111 0 0 0 50514 12193 0.000000 1.000000 0.805556 0.000000 From 915ff7b30397408812f351c2ab872fe76c598c10 Mon Sep 17 00:00:00 2001 From: Roman Babenko Date: Sat, 7 Sep 2024 08:32:18 +0300 Subject: [PATCH 36/49] upd --- meta/0f133e09.csv | 4 ++-- meta/43e1f4ea.csv | 2 +- meta/9d90ba70.csv | 2 +- meta/b081551c.csv | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/meta/0f133e09.csv b/meta/0f133e09.csv index c28e2179..d9c52a0e 100644 --- a/meta/0f133e09.csv +++ b/meta/0f133e09.csv @@ -424,8 +424,8 @@ Id,FileID,Domain,RepoName,FilePath,LineStart,LineEnd,GroundTruth,WithWords,Value 10132,f89519ab,GitHub,0f133e09,data/0f133e09/test/f89519ab.txt,10,10,F,F,,,F,F,,,,,0,0,F,F,F,Bitbucket Client ID 10135,c37b8d70,GitHub,0f133e09,data/0f133e09/test/c37b8d70.txt,10,10,F,F,,,F,F,,,,,0,0,F,F,F,Bitbucket Client ID 10136,5a7f3ab7,GitHub,0f133e09,data/0f133e09/test/5a7f3ab7.txt,10,10,T,F,226,266,F,F,,,,,0,0,F,F,F,Token -10137,983d2575,GitHub,0f133e09,data/0f133e09/test/983d2575.txt,10,10,F,F,,,F,F,,,,,0,0,F,F,F,Token -10138,0d46126a,GitHub,0f133e09,data/0f133e09/test/0d46126a.txt,10,10,F,F,,,F,F,,,,,0,0,F,F,F,Token +10137,983d2575,GitHub,0f133e09,data/0f133e09/test/983d2575.txt,10,10,T,F,210,250,F,F,,,,,0,0,F,F,F,Token +10138,0d46126a,GitHub,0f133e09,data/0f133e09/test/0d46126a.txt,10,10,T,F,226,266,F,F,,,,,0,0,F,F,F,Token 10139,5a7f3ab7,GitHub,0f133e09,data/0f133e09/test/5a7f3ab7.txt,43,43,T,F,232,272,F,F,,,,,0,0,F,F,F,Token 10140,5a7f3ab7,GitHub,0f133e09,data/0f133e09/test/5a7f3ab7.txt,21,21,T,F,232,272,F,F,,,,,0,0,F,F,F,Token 10141,5a7f3ab7,GitHub,0f133e09,data/0f133e09/test/5a7f3ab7.txt,32,32,T,F,223,263,F,F,,,,,0,0,F,F,F,Token diff --git a/meta/43e1f4ea.csv b/meta/43e1f4ea.csv index 5fe666b3..16cba278 100644 --- a/meta/43e1f4ea.csv +++ b/meta/43e1f4ea.csv @@ -19,7 +19,7 @@ Id,FileID,Domain,RepoName,FilePath,LineStart,LineEnd,GroundTruth,WithWords,Value 29435,2568f90a,GitHub,43e1f4ea,data/43e1f4ea/test/2568f90a.java,26,26,Template,T,44,48,F,F,CharsOnly,,,Secret,1.5,4,F,F,F,Password 29808,99cc63db,GitHub,43e1f4ea,data/43e1f4ea/test/99cc63db.java,27,27,Template,F,31,41,F,F,Any,,,Secret,3.12,10,F,F,F,Password 29809,57a8aa7d,GitHub,43e1f4ea,data/43e1f4ea/test/57a8aa7d.java,31,31,Template,T,31,35,F,F,CharsOnly,,,Secret,1.5,4,F,F,F,Password -29818,a6f3081b,GitHub,43e1f4ea,data/43e1f4ea/test/a6f3081b.java,47,47,Template,T,58,66,F,F,CharsOnly,,,Secret,2.75,8,F,F,F,Password +29818,a6f3081b,GitHub,43e1f4ea,data/43e1f4ea/test/a6f3081b.java,47,47,T,T,58,66,F,F,CharsOnly,,,Secret,2.75,8,F,F,F,Password 29823,dad76817,GitHub,43e1f4ea,data/43e1f4ea/test/dad76817.java,23,23,Template,T,44,57,F,F,Any,,,Secret,2.9,13,F,F,F,Password 29824,2fbdcc52,GitHub,43e1f4ea,data/43e1f4ea/test/2fbdcc52.java,16,16,Template,T,53,59,F,F,CharsOnly,,,Secret,2.25,6,F,F,F,Password 29825,317a1df2,GitHub,43e1f4ea,data/43e1f4ea/test/317a1df2.java,56,56,Template,T,52,58,F,F,CharsOnly,,,Secret,2.25,6,F,F,F,Password diff --git a/meta/9d90ba70.csv b/meta/9d90ba70.csv index 5198fb29..69d3b499 100644 --- a/meta/9d90ba70.csv +++ b/meta/9d90ba70.csv @@ -155,7 +155,7 @@ Id,FileID,Domain,RepoName,FilePath,LineStart,LineEnd,GroundTruth,WithWords,Value 30531,0d119105,GitHub,9d90ba70,data/9d90ba70/test/0d119105.py,10,10,F,F,,,T,F,CharsOnly,,,Secret,1.58,3,F,F,F,URL Credentials:Password 30532,0d119105,GitHub,9d90ba70,data/9d90ba70/test/0d119105.py,16,16,Template,F,38,41,T,F,CharsOnly,,,Secret,1.58,3,F,F,F,URL Credentials 31404,2b4070a3,GitHub,9d90ba70,data/9d90ba70/test/2b4070a3.py,24,24,Template,T,18,24,F,F,CharsOnly,,,Secret,2.25,6,F,F,F,Password -32271,0d119105,GitHub,9d90ba70,data/9d90ba70/test/0d119105.py,41,41,Template,F,33,42,T,F,Any,,,Secret,2.24,9,F,F,T,URL Credentials +32271,0d119105,GitHub,9d90ba70,data/9d90ba70/test/0d119105.py,41,41,F,F,33,42,T,F,Any,,,Secret,2.24,9,F,F,T,URL Credentials 32286,0d119105,GitHub,9d90ba70,data/9d90ba70/test/0d119105.py,34,34,Template,F,33,42,T,F,Any,,,Secret,2.24,9,F,F,T,URL Credentials 32623,1cc3b783,GitHub,9d90ba70,data/9d90ba70/src/1cc3b783.rst,3604,3604,Template,T,38,46,T,F,CharsOnly,,,Secret,2.75,8,F,F,F,URL Credentials 33783,896ef337,GitHub,9d90ba70,data/9d90ba70/test/896ef337.py,251,251,T,F,41,45,F,F,CharsOnly,,,Secret,2.0,4,F,F,F,Password diff --git a/meta/b081551c.csv b/meta/b081551c.csv index d5ec398d..0cd24a26 100644 --- a/meta/b081551c.csv +++ b/meta/b081551c.csv @@ -24,7 +24,7 @@ Id,FileID,Domain,RepoName,FilePath,LineStart,LineEnd,GroundTruth,WithWords,Value 27596,939bdae4,GitHub,b081551c,data/b081551c/test/939bdae4.py,296,296,Template,T,52,60,F,F,Any,,,Secret,2.75,8,F,F,F,Password 27652,939bdae4,GitHub,b081551c,data/b081551c/test/939bdae4.py,389,389,Template,F,37,43,T,F,NumbersOnly,,,Secret,1.58,6,F,F,F,URL Credentials 27653,939bdae4,GitHub,b081551c,data/b081551c/test/939bdae4.py,361,361,Template,F,37,43,T,F,NumbersOnly,,,Secret,1.58,6,F,F,F,URL Credentials -27667,939bdae4,GitHub,b081551c,data/b081551c/test/939bdae4.py,563,563,Template,F,40,46,T,F,Any,,,Secret,1.82,6,F,F,T,URL Credentials +27667,939bdae4,GitHub,b081551c,data/b081551c/test/939bdae4.py,563,563,F,F,40,46,T,F,Any,,,Secret,1.82,6,F,F,T,URL Credentials 27669,939bdae4,GitHub,b081551c,data/b081551c/test/939bdae4.py,515,515,Template,F,39,45,T,F,NumbersOnly,,,Secret,1.58,6,F,F,F,URL Credentials 27686,939bdae4,GitHub,b081551c,data/b081551c/test/939bdae4.py,576,576,Template,F,38,39,T,F,CharsOnly,,,Secret,0.0,1,F,F,F,Password 27729,8e43d887,GitHub,b081551c,data/b081551c/src/8e43d887.pxd,56,56,Template,T,29,37,F,F,CharsOnly,,,Secret,2.75,8,F,F,F,Auth:Password From e303092da69550cd097d93184af07d20733745f9 Mon Sep 17 00:00:00 2001 From: Roman Babenko Date: Sat, 7 Sep 2024 08:33:10 +0300 Subject: [PATCH 37/49] BM --- .ci/benchmark.txt | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/.ci/benchmark.txt b/.ci/benchmark.txt index 8088d4d0..550d98ae 100644 --- a/.ci/benchmark.txt +++ b/.ci/benchmark.txt @@ -1,5 +1,5 @@ -META MD5 4c7ae9c79f7c372e2e1a26b41b6f7932 -DATA MD5 d7ff3a303207c700d08ca47cd68a25f1 +META MD5 9715f5d8440c08adb3a3750429aab939 +DATA MD5 e0d3071e7fc3b36b6d726e759e98e416 DATA: 16345328 interested lines. MARKUP: 62817 items FileType FileNumber ValidLines Positives Negatives Templates --------------- ------------ ------------ ----------- ----------- ----------- @@ -82,7 +82,7 @@ FileType FileNumber ValidLines Positives Negatives Templat .ipynb 1 134 5 .j 1 241 4 .j2 30 5530 6 186 10 -.java 621 134132 361 1363 173 +.java 621 134132 362 1363 172 .jenkinsfile 1 58 2 6 .jinja2 1 64 2 .js 659 536413 532 2496 331 @@ -153,7 +153,7 @@ FileType FileNumber ValidLines Positives Negatives Templat .pug 2 193 2 .purs 1 69 4 .pxd 1 150 5 2 -.py 890 291553 679 3301 728 +.py 890 291553 679 3303 726 .pyi 4 1361 9 .pyp 1 167 1 .pyx 2 1094 23 @@ -209,7 +209,7 @@ FileType FileNumber ValidLines Positives Negatives Templat .ts 583 106730 157 1800 203 .tsx 54 7914 1 114 5 .ttar 1 452 1 -.txt 440 78102 5280 6361 49 +.txt 440 78102 5282 6359 49 .utf8 1 77 2 .vsixmanifest 1 36 1 .vsmdi 1 6 2 @@ -223,7 +223,7 @@ FileType FileNumber ValidLines Positives Negatives Templat .yml 419 36169 556 888 378 .zsh 6 872 12 .zsh-theme 1 97 1 -TOTAL: 10263 16345328 12193 50514 5111 +TOTAL: 10263 16345328 12196 50514 5108 credsweeper result_cnt : 0, lost_cnt : 0, true_cnt : 0, false_cnt : 0 Rules Positives Negatives Templates Reported TP FP TN FN FPR FNR ACC PRC RCL F1 ------------------------------ ----------- ----------- ----------- ---------- ---- ---- ----- ----- -------- -------- -------- ----- -------- ---- @@ -262,13 +262,13 @@ Key 3905 15720 485 Nonce 91 49 0 0 0 49 91 0.000000 1.000000 0.350000 0.000000 Other 8 8292 1 0 0 8293 8 0.000000 1.000000 0.999036 0.000000 PEM Private Key 1019 1483 0 0 0 1483 1019 0.000000 1.000000 0.592726 0.000000 -Password 1857 7538 2685 0 0 10223 1857 0.000000 1.000000 0.846275 0.000000 +Password 1858 7538 2684 0 0 10222 1858 0.000000 1.000000 0.846192 0.000000 Salt 47 76 1 0 0 77 47 0.000000 1.000000 0.620968 0.000000 Secret 1297 1575 802 0 0 2377 1297 0.000000 1.000000 0.646979 0.000000 Seed 1 6 0 0 0 6 1 0.000000 1.000000 0.857143 0.000000 Slack Token 4 1 0 0 0 1 4 0.000000 1.000000 0.200000 0.000000 -Token 638 4173 454 0 0 4627 638 0.000000 1.000000 0.878822 0.000000 +Token 640 4171 454 0 0 4625 640 0.000000 1.000000 0.878443 0.000000 Twilio API Key 0 5 2 0 0 7 0 0.000000 1.000000 -URL Credentials 210 154 218 0 0 372 210 0.000000 1.000000 0.639175 0.000000 +URL Credentials 210 156 216 0 0 372 210 0.000000 1.000000 0.639175 0.000000 UUID 1069 265 0 0 0 265 1069 0.000000 1.000000 0.198651 0.000000 - 12193 50514 5111 0 0 0 50514 12193 0.000000 1.000000 0.805556 0.000000 + 12196 50514 5108 0 0 0 50514 12196 0.000000 1.000000 0.805517 0.000000 From 75481e6c75de99a3b3569d8519cb250e79ad4f02 Mon Sep 17 00:00:00 2001 From: Roman Babenko Date: Sat, 7 Sep 2024 09:23:00 +0300 Subject: [PATCH 38/49] upd --- meta/4764adaf.csv | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/meta/4764adaf.csv b/meta/4764adaf.csv index 0426f7a9..9be5cc87 100644 --- a/meta/4764adaf.csv +++ b/meta/4764adaf.csv @@ -179,7 +179,7 @@ Id,FileID,Domain,RepoName,FilePath,LineStart,LineEnd,GroundTruth,WithWords,Value 22255,337c5eb2,GitHub,4764adaf,data/4764adaf/src/337c5eb2.json,448,448,Template,T,9,17,F,F,CharOnly,,,Secret,2.75,8,F,F,F,Password 28844,ea9346a2,GitHub,4764adaf,data/4764adaf/test/ea9346a2.json,12,12,Template,T,15,38,F,F,Any,,,Secret,2.76,23,F,F,F,Secret 29088,52c4868e,GitHub,4764adaf,data/4764adaf/test/52c4868e.js,587,587,Template,T,25,35,F,F,Any,,,Secret,3.12,10,F,F,F,Password -29344,e45e45ba,GitHub,4764adaf,data/4764adaf/src/e45e45ba.yml,43,43,Template,T,21,29,F,F,CharsOnly,,,Secret,2.75,8,F,F,F,Password +29344,e45e45ba,GitHub,4764adaf,data/4764adaf/src/e45e45ba.yml,43,43,T,T,21,29,F,F,CharsOnly,,,Secret,2.75,8,F,F,F,Password 29925,c907da53,GitHub,4764adaf,data/4764adaf/test/c907da53.js,203,203,Template,T,19,26,F,F,Any,,,Secret,2.52,7,F,F,F,Password 29926,7e37e5b4,GitHub,4764adaf,data/4764adaf/test/7e37e5b4.js,6893,6893,Template,T,27,34,F,F,Any,,,Secret,2.52,7,F,F,F,Password 30500,52c4868e,GitHub,4764adaf,data/4764adaf/test/52c4868e.js,573,573,F,F,46,56,F,F,,,,,0.0,0,F,F,F,Password From e916b9386c1238eab621158013b5eddd694b72ec Mon Sep 17 00:00:00 2001 From: Roman Babenko Date: Sat, 7 Sep 2024 09:24:04 +0300 Subject: [PATCH 39/49] BM --- .ci/benchmark.txt | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.ci/benchmark.txt b/.ci/benchmark.txt index 550d98ae..bbc0535c 100644 --- a/.ci/benchmark.txt +++ b/.ci/benchmark.txt @@ -1,5 +1,5 @@ -META MD5 9715f5d8440c08adb3a3750429aab939 -DATA MD5 e0d3071e7fc3b36b6d726e759e98e416 +META MD5 e06a3a3844c9ff7b6cbed93eee4e3c78 +DATA MD5 840a47dcbfbcc42a088dca105200a4ad DATA: 16345328 interested lines. MARKUP: 62817 items FileType FileNumber ValidLines Positives Negatives Templates --------------- ------------ ------------ ----------- ----------- ----------- @@ -220,10 +220,10 @@ FileType FileNumber ValidLines Positives Negatives Templat .xml 9 689 9 .xsl 1 311 1 .yaml 137 19004 125 345 42 -.yml 419 36169 556 888 378 +.yml 419 36169 557 888 377 .zsh 6 872 12 .zsh-theme 1 97 1 -TOTAL: 10263 16345328 12196 50514 5108 +TOTAL: 10263 16345328 12197 50514 5107 credsweeper result_cnt : 0, lost_cnt : 0, true_cnt : 0, false_cnt : 0 Rules Positives Negatives Templates Reported TP FP TN FN FPR FNR ACC PRC RCL F1 ------------------------------ ----------- ----------- ----------- ---------- ---- ---- ----- ----- -------- -------- -------- ----- -------- ---- @@ -262,7 +262,7 @@ Key 3905 15720 485 Nonce 91 49 0 0 0 49 91 0.000000 1.000000 0.350000 0.000000 Other 8 8292 1 0 0 8293 8 0.000000 1.000000 0.999036 0.000000 PEM Private Key 1019 1483 0 0 0 1483 1019 0.000000 1.000000 0.592726 0.000000 -Password 1858 7538 2684 0 0 10222 1858 0.000000 1.000000 0.846192 0.000000 +Password 1859 7538 2683 0 0 10221 1859 0.000000 1.000000 0.846109 0.000000 Salt 47 76 1 0 0 77 47 0.000000 1.000000 0.620968 0.000000 Secret 1297 1575 802 0 0 2377 1297 0.000000 1.000000 0.646979 0.000000 Seed 1 6 0 0 0 6 1 0.000000 1.000000 0.857143 0.000000 @@ -271,4 +271,4 @@ Token 640 4171 454 Twilio API Key 0 5 2 0 0 7 0 0.000000 1.000000 URL Credentials 210 156 216 0 0 372 210 0.000000 1.000000 0.639175 0.000000 UUID 1069 265 0 0 0 265 1069 0.000000 1.000000 0.198651 0.000000 - 12196 50514 5108 0 0 0 50514 12196 0.000000 1.000000 0.805517 0.000000 + 12197 50514 5107 0 0 0 50514 12197 0.000000 1.000000 0.805505 0.000000 From afba3a860fe893e8ce06498fac4f62c9dd7de5f2 Mon Sep 17 00:00:00 2001 From: Roman Babenko Date: Sat, 7 Sep 2024 11:44:31 +0300 Subject: [PATCH 40/49] [skip actions] [auxiliary] 2024-09-07T11:44:31+03:00 --- meta/eb8448ad.csv | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/meta/eb8448ad.csv b/meta/eb8448ad.csv index 17a957e2..a3a06192 100644 --- a/meta/eb8448ad.csv +++ b/meta/eb8448ad.csv @@ -24,7 +24,7 @@ Id,FileID,Domain,RepoName,FilePath,LineStart,LineEnd,GroundTruth,WithWords,Value 29841,91b3c8bc,GitHub,eb8448ad,data/eb8448ad/src/91b3c8bc.yaml,56,56,T,T,23,33,F,F,Any,,,Secret,2.46,10,F,F,F,Password 31163,e45e45ba,GitHub,eb8448ad,data/eb8448ad/src/e45e45ba.yml,61,61,T,T,26,36,F,F,Any,,,Secret,2.46,10,F,F,F,Password 31481,09c8c391,GitHub,eb8448ad,data/eb8448ad/other/09c8c391,32,32,Template,T,40,50,T,F,Any,,,Secret,2.46,10,F,F,F,URL Credentials -32303,560e5d9e,GitHub,eb8448ad,data/eb8448ad/test/560e5d9e.test,3,3,T,F,12,25,F,F,Any,,,Secret,2.74,13,F,F,F,Secret +32303,560e5d9e,GitHub,eb8448ad,data/eb8448ad/test/560e5d9e.test,3,3,F,F,12,25,F,F,Any,,,Secret,2.74,13,F,F,F,Secret 32570,6eb51d25,GitHub,eb8448ad,data/eb8448ad/src/6eb51d25.sh,18,18,F,T,50,61,T,F,Any,,,Secret,2.78,11,F,F,F,Password:URL Credentials 137862,6eb51d25,GitHub,eb8448ad,data/eb8448ad/src/6eb51d25.sh,18,18,T,F,158,168,F,F,,,,,0.0,0,F,F,F,Password:URL Credentials 33535,09c8c391,GitHub,eb8448ad,data/eb8448ad/other/09c8c391,43,43,T,F,18,134,F,F,Any,,JSON Web Token,Token,5.23,116,F,F,F,JSON Web Token:Token From 568bd58eeed79b60ff224864b6fce9fcbb671649 Mon Sep 17 00:00:00 2001 From: Roman Babenko Date: Sat, 7 Sep 2024 11:45:41 +0300 Subject: [PATCH 41/49] F --- .ci/benchmark.txt | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.ci/benchmark.txt b/.ci/benchmark.txt index bbc0535c..82c23932 100644 --- a/.ci/benchmark.txt +++ b/.ci/benchmark.txt @@ -1,5 +1,5 @@ -META MD5 e06a3a3844c9ff7b6cbed93eee4e3c78 -DATA MD5 840a47dcbfbcc42a088dca105200a4ad +META MD5 088b8c22fe9cee19ff3ae4e184a78ca0 +DATA MD5 2cef557d176eccea34bc2093c2d4f893 DATA: 16345328 interested lines. MARKUP: 62817 items FileType FileNumber ValidLines Positives Negatives Templates --------------- ------------ ------------ ----------- ----------- ----------- @@ -195,7 +195,7 @@ FileType FileNumber ValidLines Positives Negatives Templat .t 9 1767 25 44 13 .td 2 14002 6 .template 19 1633 4 35 11 -.test 2 24 25 4 +.test 2 24 24 1 4 .testsettings 1 21 1 10 .tf 21 1377 2 30 2 .tfstate 4 307 22 11 4 @@ -223,7 +223,7 @@ FileType FileNumber ValidLines Positives Negatives Templat .yml 419 36169 557 888 377 .zsh 6 872 12 .zsh-theme 1 97 1 -TOTAL: 10263 16345328 12197 50514 5107 +TOTAL: 10263 16345328 12196 50515 5107 credsweeper result_cnt : 0, lost_cnt : 0, true_cnt : 0, false_cnt : 0 Rules Positives Negatives Templates Reported TP FP TN FN FPR FNR ACC PRC RCL F1 ------------------------------ ----------- ----------- ----------- ---------- ---- ---- ----- ----- -------- -------- -------- ----- -------- ---- @@ -264,11 +264,11 @@ Other 8 8292 1 PEM Private Key 1019 1483 0 0 0 1483 1019 0.000000 1.000000 0.592726 0.000000 Password 1859 7538 2683 0 0 10221 1859 0.000000 1.000000 0.846109 0.000000 Salt 47 76 1 0 0 77 47 0.000000 1.000000 0.620968 0.000000 -Secret 1297 1575 802 0 0 2377 1297 0.000000 1.000000 0.646979 0.000000 +Secret 1296 1576 802 0 0 2378 1296 0.000000 1.000000 0.647251 0.000000 Seed 1 6 0 0 0 6 1 0.000000 1.000000 0.857143 0.000000 Slack Token 4 1 0 0 0 1 4 0.000000 1.000000 0.200000 0.000000 Token 640 4171 454 0 0 4625 640 0.000000 1.000000 0.878443 0.000000 Twilio API Key 0 5 2 0 0 7 0 0.000000 1.000000 URL Credentials 210 156 216 0 0 372 210 0.000000 1.000000 0.639175 0.000000 UUID 1069 265 0 0 0 265 1069 0.000000 1.000000 0.198651 0.000000 - 12197 50514 5107 0 0 0 50514 12197 0.000000 1.000000 0.805505 0.000000 + 12196 50515 5107 0 0 0 50515 12196 0.000000 1.000000 0.805521 0.000000 From 267ea797e166286e3e4272255d1cc7019a76d0ea Mon Sep 17 00:00:00 2001 From: Roman Babenko Date: Sat, 7 Sep 2024 16:40:13 +0300 Subject: [PATCH 42/49] upd --- meta/50595139.csv | 2 ++ meta/a09d9e50.csv | 1 + meta/c5867a40.csv | 2 +- 3 files changed, 4 insertions(+), 1 deletion(-) diff --git a/meta/50595139.csv b/meta/50595139.csv index 1557b398..ac6f8696 100644 --- a/meta/50595139.csv +++ b/meta/50595139.csv @@ -270,3 +270,5 @@ Id,FileID,Domain,RepoName,FilePath,LineStart,LineEnd,GroundTruth,WithWords,Value 1101762,28de09ab,GitHub,50595139,data/50595139/test/28de09ab.m,83,83,F,F,,,F,F,,,,,0.00,,F,F,F,Password 1101763,28de09ab,GitHub,50595139,data/50595139/test/28de09ab.m,105,105,F,F,,,F,F,,,,,0.00,,F,F,F,Password 1101764,28de09ab,GitHub,50595139,data/50595139/test/28de09ab.m,296,296,F,F,,,F,F,,,,,0.00,,F,F,F,Password +1479625,28de09ab,GitHub,50595139,data/50595139/test/28de09ab.m,267,267,T,F,107,118,F,F,,,,,0.0,0,F,F,F,Password +1479626,28de09ab,GitHub,50595139,data/50595139/test/28de09ab.m,654,654,T,F,27,33,F,F,,,,,0.0,0,F,F,F,Password diff --git a/meta/a09d9e50.csv b/meta/a09d9e50.csv index 27341dde..2c516d40 100644 --- a/meta/a09d9e50.csv +++ b/meta/a09d9e50.csv @@ -409,3 +409,4 @@ Id,FileID,Domain,RepoName,FilePath,LineStart,LineEnd,GroundTruth,WithWords,Value 1479408,054074fe,GitHub,a09d9e50,data/a09d9e50/test/054074fe.ndjson,1404,1404,T,F,431,441,F,F,,,,,0.0,0,F,F,F,CMD ConvertTo-SecureString 1479596,054074fe,GitHub,a09d9e50,data/a09d9e50/test/054074fe.ndjson,367,367,F,F,1851,1867,F,F,,,,,0.0,0,F,F,F,Key:Token 1479623,054074fe,GitHub,a09d9e50,data/a09d9e50/test/054074fe.ndjson,3266,3266,F,F,,,F,F,,,,,0.0,0,F,F,F,Password +1479627,054074fe,GitHub,a09d9e50,data/a09d9e50/test/054074fe.ndjson,3604,3604,T,F,450,500,F,F,,,,RapidAPI,0.0,0,F,F,F,Key diff --git a/meta/c5867a40.csv b/meta/c5867a40.csv index 8778af53..11636f87 100644 --- a/meta/c5867a40.csv +++ b/meta/c5867a40.csv @@ -86,7 +86,7 @@ Id,FileID,Domain,RepoName,FilePath,LineStart,LineEnd,GroundTruth,WithWords,Value 25332,c04c9d85,GitHub,c5867a40,data/c5867a40/src/c04c9d85.rrc,30,30,F,F,,,F,F,,,,,0,0,F,F,F,Secret 25333,7346d444,GitHub,c5867a40,data/c5867a40/src/7346d444.rrc,30,30,F,F,,,F,F,,,,,0,0,F,F,F,Secret 26339,607d8792,GitHub,c5867a40,data/c5867a40/src/607d8792.m,42,42,F,F,-1,-1,F,F,,,,,0.0,0,F,F,F,Other -27528,7975d392,GitHub,c5867a40,data/c5867a40/test/7975d392.m,52,52,T,F,47,73,F,F,Any,,,Secret,0,1,F,F,F,Password +27528,7975d392,GitHub,c5867a40,data/c5867a40/test/7975d392.m,52,52,T,F,47,73,F,F,Any,,,Secret,0,1,F,F,F,Secret 27607,81820b04,GitHub,c5867a40,data/c5867a40/src/81820b04.java,38,38,F,T,39,45,F,F,CharsOnly,,,Secret,2.25,6,F,F,F,Secret 27902,27af74d2,GitHub,c5867a40,data/c5867a40/src/27af74d2.java,27,27,F,F,22,54,F,F,,,,,0.0,0,F,F,F,Bitbucket Client ID 27903,0d282823,GitHub,c5867a40,data/c5867a40/src/0d282823.java,28,28,F,F,22,54,F,F,,,,,0.0,0,F,F,F,Bitbucket Client ID From 74449b4c4a60bd20f33ae8d7c8f10bfd7db5b1d1 Mon Sep 17 00:00:00 2001 From: Roman Babenko Date: Sat, 7 Sep 2024 16:42:24 +0300 Subject: [PATCH 43/49] updBM --- .ci/benchmark.txt | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/.ci/benchmark.txt b/.ci/benchmark.txt index 82c23932..986481aa 100644 --- a/.ci/benchmark.txt +++ b/.ci/benchmark.txt @@ -1,6 +1,6 @@ -META MD5 088b8c22fe9cee19ff3ae4e184a78ca0 -DATA MD5 2cef557d176eccea34bc2093c2d4f893 -DATA: 16345328 interested lines. MARKUP: 62817 items +META MD5 0f71add1b9f335f6f9f5a248190dfba7 +DATA MD5 3e97ea39165fda673fb825e9523e02be +DATA: 16345328 interested lines. MARKUP: 62820 items FileType FileNumber ValidLines Positives Negatives Templates --------------- ------------ ------------ ----------- ----------- ----------- 194 28318 71 417 90 @@ -108,7 +108,7 @@ FileType FileNumber ValidLines Positives Negatives Templat .lock 24 160912 142 .log 2 199 38 52 .lua 10 1924 37 3 -.m 16 13358 10 159 3 +.m 16 13358 12 159 3 .manifest 3 102 9 6 .markdown 3 139 3 1 .markerb 3 12 3 @@ -125,7 +125,7 @@ FileType FileNumber ValidLines Positives Negatives Templat .mqh 1 1023 2 .msg 1 26644 1 1 .mysql 1 36 2 -.ndjson 2 5006 74 242 2 +.ndjson 2 5006 75 242 2 .nix 4 211 12 .nolint 1 2 1 .odd 1 1281 43 @@ -223,7 +223,7 @@ FileType FileNumber ValidLines Positives Negatives Templat .yml 419 36169 557 888 377 .zsh 6 872 12 .zsh-theme 1 97 1 -TOTAL: 10263 16345328 12196 50515 5107 +TOTAL: 10263 16345328 12199 50515 5107 credsweeper result_cnt : 0, lost_cnt : 0, true_cnt : 0, false_cnt : 0 Rules Positives Negatives Templates Reported TP FP TN FN FPR FNR ACC PRC RCL F1 ------------------------------ ----------- ----------- ----------- ---------- ---- ---- ----- ----- -------- -------- -------- ----- -------- ---- @@ -258,17 +258,17 @@ Grafana Provisioned API Key 22 1 0 JSON Web Token 170 61 0 0 0 61 170 0.000000 1.000000 0.264069 0.000000 Jira / Confluence PAT token 0 4 0 0 0 4 0 0.000000 1.000000 Jira 2FA 15 6 1 0 0 7 15 0.000000 1.000000 0.318182 0.000000 -Key 3905 15720 485 0 0 16205 3905 0.000000 1.000000 0.805818 0.000000 +Key 3906 15720 485 0 0 16205 3906 0.000000 1.000000 0.805778 0.000000 Nonce 91 49 0 0 0 49 91 0.000000 1.000000 0.350000 0.000000 Other 8 8292 1 0 0 8293 8 0.000000 1.000000 0.999036 0.000000 PEM Private Key 1019 1483 0 0 0 1483 1019 0.000000 1.000000 0.592726 0.000000 -Password 1859 7538 2683 0 0 10221 1859 0.000000 1.000000 0.846109 0.000000 +Password 1860 7538 2683 0 0 10221 1860 0.000000 1.000000 0.846039 0.000000 Salt 47 76 1 0 0 77 47 0.000000 1.000000 0.620968 0.000000 -Secret 1296 1576 802 0 0 2378 1296 0.000000 1.000000 0.647251 0.000000 +Secret 1297 1576 802 0 0 2378 1297 0.000000 1.000000 0.647075 0.000000 Seed 1 6 0 0 0 6 1 0.000000 1.000000 0.857143 0.000000 Slack Token 4 1 0 0 0 1 4 0.000000 1.000000 0.200000 0.000000 Token 640 4171 454 0 0 4625 640 0.000000 1.000000 0.878443 0.000000 Twilio API Key 0 5 2 0 0 7 0 0.000000 1.000000 URL Credentials 210 156 216 0 0 372 210 0.000000 1.000000 0.639175 0.000000 UUID 1069 265 0 0 0 265 1069 0.000000 1.000000 0.198651 0.000000 - 12196 50515 5107 0 0 0 50515 12196 0.000000 1.000000 0.805521 0.000000 + 12199 50515 5107 0 0 0 50515 12199 0.000000 1.000000 0.805482 0.000000 From d1cad4ae12296072ffe3b7bd1f394eba7d566f3c Mon Sep 17 00:00:00 2001 From: Roman Babenko Date: Sat, 7 Sep 2024 17:07:26 +0300 Subject: [PATCH 44/49] upd --- .ci/benchmark.txt | 20 ++++++++++---------- meta/0f133e09.csv | 4 ++-- meta/387016a6.csv | 2 +- meta/cc51a2f0.csv | 1 + 4 files changed, 14 insertions(+), 13 deletions(-) diff --git a/.ci/benchmark.txt b/.ci/benchmark.txt index 986481aa..d702cd4b 100644 --- a/.ci/benchmark.txt +++ b/.ci/benchmark.txt @@ -1,9 +1,9 @@ -META MD5 0f71add1b9f335f6f9f5a248190dfba7 -DATA MD5 3e97ea39165fda673fb825e9523e02be -DATA: 16345328 interested lines. MARKUP: 62820 items +META MD5 63be98f7cfaf83aa33052540ec6f1420 +DATA MD5 c10fc577d910543c506fa17a8a63bfe7 +DATA: 16345328 interested lines. MARKUP: 62821 items FileType FileNumber ValidLines Positives Negatives Templates --------------- ------------ ------------ ----------- ----------- ----------- - 194 28318 71 417 90 + 194 28318 71 418 90 .1 2 641 2 5 .admx 1 26 1 .adoc 1 158 13 6 1 @@ -209,7 +209,7 @@ FileType FileNumber ValidLines Positives Negatives Templat .ts 583 106730 157 1800 203 .tsx 54 7914 1 114 5 .ttar 1 452 1 -.txt 440 78102 5282 6359 49 +.txt 440 78102 5284 6357 49 .utf8 1 77 2 .vsixmanifest 1 36 1 .vsmdi 1 6 2 @@ -220,10 +220,10 @@ FileType FileNumber ValidLines Positives Negatives Templat .xml 9 689 9 .xsl 1 311 1 .yaml 137 19004 125 345 42 -.yml 419 36169 557 888 377 +.yml 419 36169 558 887 377 .zsh 6 872 12 .zsh-theme 1 97 1 -TOTAL: 10263 16345328 12199 50515 5107 +TOTAL: 10263 16345328 12202 50513 5107 credsweeper result_cnt : 0, lost_cnt : 0, true_cnt : 0, false_cnt : 0 Rules Positives Negatives Templates Reported TP FP TN FN FPR FNR ACC PRC RCL F1 ------------------------------ ----------- ----------- ----------- ---------- ---- ---- ----- ----- -------- -------- -------- ----- -------- ---- @@ -262,13 +262,13 @@ Key 3906 15720 485 Nonce 91 49 0 0 0 49 91 0.000000 1.000000 0.350000 0.000000 Other 8 8292 1 0 0 8293 8 0.000000 1.000000 0.999036 0.000000 PEM Private Key 1019 1483 0 0 0 1483 1019 0.000000 1.000000 0.592726 0.000000 -Password 1860 7538 2683 0 0 10221 1860 0.000000 1.000000 0.846039 0.000000 +Password 1860 7539 2683 0 0 10222 1860 0.000000 1.000000 0.846052 0.000000 Salt 47 76 1 0 0 77 47 0.000000 1.000000 0.620968 0.000000 Secret 1297 1576 802 0 0 2378 1297 0.000000 1.000000 0.647075 0.000000 Seed 1 6 0 0 0 6 1 0.000000 1.000000 0.857143 0.000000 Slack Token 4 1 0 0 0 1 4 0.000000 1.000000 0.200000 0.000000 -Token 640 4171 454 0 0 4625 640 0.000000 1.000000 0.878443 0.000000 +Token 643 4168 454 0 0 4622 643 0.000000 1.000000 0.877873 0.000000 Twilio API Key 0 5 2 0 0 7 0 0.000000 1.000000 URL Credentials 210 156 216 0 0 372 210 0.000000 1.000000 0.639175 0.000000 UUID 1069 265 0 0 0 265 1069 0.000000 1.000000 0.198651 0.000000 - 12199 50515 5107 0 0 0 50515 12199 0.000000 1.000000 0.805482 0.000000 + 12202 50513 5107 0 0 0 50513 12202 0.000000 1.000000 0.805437 0.000000 diff --git a/meta/0f133e09.csv b/meta/0f133e09.csv index d9c52a0e..dd561d8e 100644 --- a/meta/0f133e09.csv +++ b/meta/0f133e09.csv @@ -429,8 +429,8 @@ Id,FileID,Domain,RepoName,FilePath,LineStart,LineEnd,GroundTruth,WithWords,Value 10139,5a7f3ab7,GitHub,0f133e09,data/0f133e09/test/5a7f3ab7.txt,43,43,T,F,232,272,F,F,,,,,0,0,F,F,F,Token 10140,5a7f3ab7,GitHub,0f133e09,data/0f133e09/test/5a7f3ab7.txt,21,21,T,F,232,272,F,F,,,,,0,0,F,F,F,Token 10141,5a7f3ab7,GitHub,0f133e09,data/0f133e09/test/5a7f3ab7.txt,32,32,T,F,223,263,F,F,,,,,0,0,F,F,F,Token -10142,4bc35222,GitHub,0f133e09,data/0f133e09/test/4bc35222.txt,10,10,F,F,,,F,F,,,,,0,0,F,F,F,Token -10143,6ec6580c,GitHub,0f133e09,data/0f133e09/test/6ec6580c.txt,10,10,F,F,,,F,F,,,,,0,0,F,F,F,Token +10142,4bc35222,GitHub,0f133e09,data/0f133e09/test/4bc35222.txt,10,10,T,F,209,249,F,F,,,,,0,0,F,F,F,Token +10143,6ec6580c,GitHub,0f133e09,data/0f133e09/test/6ec6580c.txt,10,10,T,F,209249,,F,F,,,,,0,0,F,F,F,Token 10149,59a5a023,GitHub,0f133e09,data/0f133e09/test/59a5a023.txt,21,21,F,F,,,F,F,,,,,0,0,F,F,F,Bitbucket Client ID 10150,ad1d7326,GitHub,0f133e09,data/0f133e09/test/ad1d7326.txt,21,21,F,F,,,F,F,,,,,0,0,F,F,F,Bitbucket Client ID 10151,2ddf0fb2,GitHub,0f133e09,data/0f133e09/test/2ddf0fb2.txt,10,10,F,F,,,F,F,,,,,0,0,F,F,F,Bitbucket Client ID diff --git a/meta/387016a6.csv b/meta/387016a6.csv index c2f022fc..406b76a6 100644 --- a/meta/387016a6.csv +++ b/meta/387016a6.csv @@ -88,7 +88,7 @@ Id,FileID,Domain,RepoName,FilePath,LineStart,LineEnd,GroundTruth,WithWords,Value 17780,3b2d8e87,GitHub,387016a6,data/387016a6/src/3b2d8e87.gradle,308,308,F,F,,,F,F,,,,,0,0,F,F,F,API 17801,b72eb970,GitHub,387016a6,data/387016a6/test/b72eb970.gradle,34,34,F,F,,,F,F,,,,,0,0,F,F,F,API 17836,27d88c46,GitHub,387016a6,data/387016a6/other/27d88c46.md,3,3,F,F,,,F,F,,,,,0,0,F,F,F,API -18460,478a6429,GitHub,387016a6,data/387016a6/test/478a6429.yml,37,37,F,F,,,F,F,,,,,0,0,F,F,F,Token +18460,478a6429,GitHub,387016a6,data/387016a6/test/478a6429.yml,37,37,T,F,60,76,F,F,,,,,0,0,F,F,F,Token 18461,ea86b6b8,GitHub,387016a6,data/387016a6/test/ea86b6b8.yml,18,18,F,F,,,F,F,,,,,0,0,F,F,F,Token 18462,ea86b6b8,GitHub,387016a6,data/387016a6/test/ea86b6b8.yml,35,35,F,F,,,F,F,,,,,0,0,F,F,F,Token 18474,1d5b0bff,GitHub,387016a6,data/387016a6/test/1d5b0bff.key,15,34,T,T,,,F,F,Any,Private,,Unknown,2.59,31,F,F,F,PEM Private Key diff --git a/meta/cc51a2f0.csv b/meta/cc51a2f0.csv index 5cb54013..3066a65a 100644 --- a/meta/cc51a2f0.csv +++ b/meta/cc51a2f0.csv @@ -136,3 +136,4 @@ Id,FileID,Domain,RepoName,FilePath,LineStart,LineEnd,GroundTruth,WithWords,Value 134598,c65d41b8,GitHub,cc51a2f0,data/cc51a2f0/test/c65d41b8.bats,971,971,F,F,,,F,F,,,,,0,0,F,F,F,Auth 1238228,05663f5e,GitHub,cc51a2f0,data/cc51a2f0/src/05663f5e.sh,161,161,F,F,,,F,F,,,,,0.0,0,F,F,F,Password 1238229,141cf316,GitHub,cc51a2f0,data/cc51a2f0/src/141cf316.sh,378,378,F,F,,,F,F,,,,,0.0,0,F,F,F,Password +1013310,92d2047e,GitHub,cc51a2f0,data/cc51a2f0/other/92d2047e,40,40,F,F,,,F,F,,,,,0,0,F,F,F,Password From b8dee88550126a143d1f19f8f689b8d73ed74e54 Mon Sep 17 00:00:00 2001 From: Roman Babenko Date: Sat, 7 Sep 2024 18:03:05 +0300 Subject: [PATCH 45/49] [skip actions] [auxiliary] 2024-09-07T18:03:05+03:00 --- meta/0f133e09.csv | 2 +- meta/50595139.csv | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/meta/0f133e09.csv b/meta/0f133e09.csv index dd561d8e..2370befc 100644 --- a/meta/0f133e09.csv +++ b/meta/0f133e09.csv @@ -430,7 +430,7 @@ Id,FileID,Domain,RepoName,FilePath,LineStart,LineEnd,GroundTruth,WithWords,Value 10140,5a7f3ab7,GitHub,0f133e09,data/0f133e09/test/5a7f3ab7.txt,21,21,T,F,232,272,F,F,,,,,0,0,F,F,F,Token 10141,5a7f3ab7,GitHub,0f133e09,data/0f133e09/test/5a7f3ab7.txt,32,32,T,F,223,263,F,F,,,,,0,0,F,F,F,Token 10142,4bc35222,GitHub,0f133e09,data/0f133e09/test/4bc35222.txt,10,10,T,F,209,249,F,F,,,,,0,0,F,F,F,Token -10143,6ec6580c,GitHub,0f133e09,data/0f133e09/test/6ec6580c.txt,10,10,T,F,209249,,F,F,,,,,0,0,F,F,F,Token +10143,6ec6580c,GitHub,0f133e09,data/0f133e09/test/6ec6580c.txt,10,10,T,F,209,249,F,F,,,,,0,0,F,F,F,Token 10149,59a5a023,GitHub,0f133e09,data/0f133e09/test/59a5a023.txt,21,21,F,F,,,F,F,,,,,0,0,F,F,F,Bitbucket Client ID 10150,ad1d7326,GitHub,0f133e09,data/0f133e09/test/ad1d7326.txt,21,21,F,F,,,F,F,,,,,0,0,F,F,F,Bitbucket Client ID 10151,2ddf0fb2,GitHub,0f133e09,data/0f133e09/test/2ddf0fb2.txt,10,10,F,F,,,F,F,,,,,0,0,F,F,F,Bitbucket Client ID diff --git a/meta/50595139.csv b/meta/50595139.csv index ac6f8696..93f86235 100644 --- a/meta/50595139.csv +++ b/meta/50595139.csv @@ -272,3 +272,5 @@ Id,FileID,Domain,RepoName,FilePath,LineStart,LineEnd,GroundTruth,WithWords,Value 1101764,28de09ab,GitHub,50595139,data/50595139/test/28de09ab.m,296,296,F,F,,,F,F,,,,,0.00,,F,F,F,Password 1479625,28de09ab,GitHub,50595139,data/50595139/test/28de09ab.m,267,267,T,F,107,118,F,F,,,,,0.0,0,F,F,F,Password 1479626,28de09ab,GitHub,50595139,data/50595139/test/28de09ab.m,654,654,T,F,27,33,F,F,,,,,0.0,0,F,F,F,Password +1479629,28de09ab,GitHub,50595139,data/50595139/test/28de09ab.m,233,233,T,F,107,115,F,F,,,,,0.0,0,F,F,F,Password +1479630,28de09ab,GitHub,50595139,data/50595139/test/28de09ab.m,250,250,T,F,107,115,F,F,,,,,0.0,0,F,F,F,Password From a59ae8a8e8a215667618f25ed1609bd69a8be722 Mon Sep 17 00:00:00 2001 From: Roman Babenko Date: Sat, 7 Sep 2024 18:03:51 +0300 Subject: [PATCH 46/49] fix --- .ci/benchmark.txt | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.ci/benchmark.txt b/.ci/benchmark.txt index d702cd4b..438bfdc7 100644 --- a/.ci/benchmark.txt +++ b/.ci/benchmark.txt @@ -1,6 +1,6 @@ -META MD5 63be98f7cfaf83aa33052540ec6f1420 -DATA MD5 c10fc577d910543c506fa17a8a63bfe7 -DATA: 16345328 interested lines. MARKUP: 62821 items +META MD5 14abeb28740ba716f72f0a884e989d42 +DATA MD5 e1c9c3ef5c31bd8123fba2a8df0e6729 +DATA: 16345328 interested lines. MARKUP: 62823 items FileType FileNumber ValidLines Positives Negatives Templates --------------- ------------ ------------ ----------- ----------- ----------- 194 28318 71 418 90 @@ -108,7 +108,7 @@ FileType FileNumber ValidLines Positives Negatives Templat .lock 24 160912 142 .log 2 199 38 52 .lua 10 1924 37 3 -.m 16 13358 12 159 3 +.m 16 13358 14 159 3 .manifest 3 102 9 6 .markdown 3 139 3 1 .markerb 3 12 3 @@ -223,7 +223,7 @@ FileType FileNumber ValidLines Positives Negatives Templat .yml 419 36169 558 887 377 .zsh 6 872 12 .zsh-theme 1 97 1 -TOTAL: 10263 16345328 12202 50513 5107 +TOTAL: 10263 16345328 12204 50513 5107 credsweeper result_cnt : 0, lost_cnt : 0, true_cnt : 0, false_cnt : 0 Rules Positives Negatives Templates Reported TP FP TN FN FPR FNR ACC PRC RCL F1 ------------------------------ ----------- ----------- ----------- ---------- ---- ---- ----- ----- -------- -------- -------- ----- -------- ---- @@ -262,7 +262,7 @@ Key 3906 15720 485 Nonce 91 49 0 0 0 49 91 0.000000 1.000000 0.350000 0.000000 Other 8 8292 1 0 0 8293 8 0.000000 1.000000 0.999036 0.000000 PEM Private Key 1019 1483 0 0 0 1483 1019 0.000000 1.000000 0.592726 0.000000 -Password 1860 7539 2683 0 0 10222 1860 0.000000 1.000000 0.846052 0.000000 +Password 1862 7539 2683 0 0 10222 1862 0.000000 1.000000 0.845912 0.000000 Salt 47 76 1 0 0 77 47 0.000000 1.000000 0.620968 0.000000 Secret 1297 1576 802 0 0 2378 1297 0.000000 1.000000 0.647075 0.000000 Seed 1 6 0 0 0 6 1 0.000000 1.000000 0.857143 0.000000 @@ -271,4 +271,4 @@ Token 643 4168 454 Twilio API Key 0 5 2 0 0 7 0 0.000000 1.000000 URL Credentials 210 156 216 0 0 372 210 0.000000 1.000000 0.639175 0.000000 UUID 1069 265 0 0 0 265 1069 0.000000 1.000000 0.198651 0.000000 - 12202 50513 5107 0 0 0 50513 12202 0.000000 1.000000 0.805437 0.000000 + 12204 50513 5107 0 0 0 50513 12204 0.000000 1.000000 0.805412 0.000000 From 9e8c67675f518612d7e8750f3a43d9e25eb28589 Mon Sep 17 00:00:00 2001 From: Roman Babenko Date: Sun, 8 Sep 2024 10:57:44 +0300 Subject: [PATCH 47/49] upd --- .ci/benchmark.txt | 4 ++-- meta/2ba83c6a.csv | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.ci/benchmark.txt b/.ci/benchmark.txt index 438bfdc7..716dcf46 100644 --- a/.ci/benchmark.txt +++ b/.ci/benchmark.txt @@ -1,5 +1,5 @@ -META MD5 14abeb28740ba716f72f0a884e989d42 -DATA MD5 e1c9c3ef5c31bd8123fba2a8df0e6729 +META MD5 a384744d4dba5026f787a85f77a77b62 +DATA MD5 c562bf518eda436ca5a530c176aa0338 DATA: 16345328 interested lines. MARKUP: 62823 items FileType FileNumber ValidLines Positives Negatives Templates --------------- ------------ ------------ ----------- ----------- ----------- diff --git a/meta/2ba83c6a.csv b/meta/2ba83c6a.csv index c483a76b..cc4fc545 100644 --- a/meta/2ba83c6a.csv +++ b/meta/2ba83c6a.csv @@ -5847,8 +5847,8 @@ Id,FileID,Domain,RepoName,FilePath,LineStart,LineEnd,GroundTruth,WithWords,Value 61137,911dde03,GitHub,2ba83c6a,data/2ba83c6a/test/911dde03.txt,19134,19134,T,F,6,54,F,F,,,,,0.0,0,F,F,F,Key 61138,911dde03,GitHub,2ba83c6a,data/2ba83c6a/test/911dde03.txt,19142,19142,T,F,6,54,F,F,,,,,0.0,0,F,F,F,Key 61139,911dde03,GitHub,2ba83c6a,data/2ba83c6a/test/911dde03.txt,19150,19150,T,F,6,54,F,F,,,,,0.0,0,F,F,F,Key -61140,5e763eae,GitHub,2ba83c6a,data/2ba83c6a/test/5e763eae.txt,237,237,T,F,6,-1,F,F,,,,,0.0,0,F,F,F,Key -61141,08d3f9a3,GitHub,2ba83c6a,data/2ba83c6a/test/08d3f9a3.txt,185,185,T,F,6,-1,F,F,,,,,0.0,0,F,F,F,Key +61140,5e763eae,GitHub,2ba83c6a,data/2ba83c6a/test/5e763eae.txt,237,237,T,F,6,38,F,F,,,,,0.0,0,F,F,F,Key +61141,08d3f9a3,GitHub,2ba83c6a,data/2ba83c6a/test/08d3f9a3.txt,185,185,T,F,6,38,F,F,,,,,0.0,0,F,F,F,Key 61142,911dde03,GitHub,2ba83c6a,data/2ba83c6a/test/911dde03.txt,9676,9676,T,F,6,54,F,F,,,,,0.0,0,F,F,F,Key 61143,911dde03,GitHub,2ba83c6a,data/2ba83c6a/test/911dde03.txt,9684,9684,T,F,6,54,F,F,,,,,0.0,0,F,F,F,Key 61144,911dde03,GitHub,2ba83c6a,data/2ba83c6a/test/911dde03.txt,9692,9692,T,F,6,54,F,F,,,,,0.0,0,F,F,F,Key From 69822c2fa3c3d21ef9234171be200692afd2613c Mon Sep 17 00:00:00 2001 From: Roman Babenko Date: Sun, 8 Sep 2024 13:36:31 +0300 Subject: [PATCH 48/49] [skip actions] [auxiliary] 2024-09-08T13:36:31+03:00 --- download_data.py | 1 + meta/1dc6a932.csv | 8 -------- meta/35c273e2.csv | 1 - 3 files changed, 1 insertion(+), 9 deletions(-) diff --git a/download_data.py b/download_data.py index af8a1a73..1a70d6f3 100644 --- a/download_data.py +++ b/download_data.py @@ -145,6 +145,7 @@ def move_files(temp_dir, dataset_dir): if key in interesting_files: # check correctness assert interesting_files[key] == file_path, (key, file_path) + assert not file_path.endswith(".xml"), f"xml parsing breaks raw text numeration {file_path}" else: interesting_files[key] = file_path diff --git a/meta/1dc6a932.csv b/meta/1dc6a932.csv index a3c34311..b897edf6 100644 --- a/meta/1dc6a932.csv +++ b/meta/1dc6a932.csv @@ -8,11 +8,3 @@ Id,FileID,Domain,RepoName,FilePath,LineStart,LineEnd,GroundTruth,WithWords,Value 70860,47f64110,GitHub,1dc6a932,data/1dc6a932/src/47f64110.py,192,192,F,F,31,35,F,F,,,,,1.5,4,F,F,F,Auth 70861,47f64110,GitHub,1dc6a932,data/1dc6a932/src/47f64110.py,196,196,F,F,31,35,F,F,,,,,1.0,4,F,F,F,Auth 70862,47f64110,GitHub,1dc6a932,data/1dc6a932/src/47f64110.py,204,204,F,F,33,37,F,F,,,,,1.0,4,F,F,F,Auth -109428,049185df,GitHub,1dc6a932,data/1dc6a932/src/049185df.xml,12,12,F,F,30,47,F,F,,,,,3.3027227,-1,F,F,F,Password -109429,0641a548,GitHub,1dc6a932,data/1dc6a932/src/0641a548.xml,5,5,F,F,27,35,F,F,,,,,3.121928,-1,F,F,F,Password -109430,64b0710b,GitHub,1dc6a932,data/1dc6a932/src/64b0710b.xml,12,12,F,F,31,45,F,F,,,,,3.625,-1,F,F,F,Password -109431,8d7e5b82,GitHub,1dc6a932,data/1dc6a932/src/8d7e5b82.xml,11,11,F,F,31,44,F,F,,,,,3.3232315,-1,F,F,F,Password -109432,9d94c32e,GitHub,1dc6a932,data/1dc6a932/src/9d94c32e.xml,5,5,F,F,27,44,F,F,,,,,3.5766177,-1,F,F,F,Password -109433,bc36bd21,GitHub,1dc6a932,data/1dc6a932/src/bc36bd21.xml,7,7,F,F,29,43,F,F,,,,,3.375,-1,F,F,F,Password -109434,bce54dc5,GitHub,1dc6a932,data/1dc6a932/src/bce54dc5.xml,12,12,F,F,31,41,F,F,,,,,3.0849626,-1,F,F,F,Password -109435,dcb716c6,GitHub,1dc6a932,data/1dc6a932/src/dcb716c6.xml,11,11,F,F,31,39,F,F,,,,,2.9219282,-1,F,F,F,Password diff --git a/meta/35c273e2.csv b/meta/35c273e2.csv index a434fc24..8bd2c0d0 100644 --- a/meta/35c273e2.csv +++ b/meta/35c273e2.csv @@ -13,7 +13,6 @@ Id,FileID,Domain,RepoName,FilePath,LineStart,LineEnd,GroundTruth,WithWords,Value 84942,5458647e,GitHub,35c273e2,data/35c273e2/src/5458647e.java,81,81,F,F,78,92,F,F,,,,,2.64,14,F,F,F,Token 84943,5458647e,GitHub,35c273e2,data/35c273e2/src/5458647e.java,89,89,F,F,70,74,F,F,,,,,2.0,4,F,F,F,Token 84944,7849399f,GitHub,35c273e2,data/35c273e2/src/7849399f.java,155,155,F,F,36,40,F,F,,,,,2.0,4,F,F,F,Credential -84945,9c5fb3d1,GitHub,35c273e2,data/35c273e2/src/9c5fb3d1.xml,12,12,F,F,169,189,F,F,,,,,3.17,20,F,F,F,Credential 84946,29edcedd,GitHub,35c273e2,data/35c273e2/test/29edcedd.java,27,27,T,F,45,81,F,F,,,,,3.51,36,F,F,F,Token:UUID 84947,470893ef,GitHub,35c273e2,data/35c273e2/test/470893ef.java,27,27,F,F,26,35,F,F,,,,,2.6,11,F,F,F,Password 84948,6e4997a7,GitHub,35c273e2,data/35c273e2/test/6e4997a7.groovy,34,34,T,F,48,84,F,F,,,,,3.29,36,F,F,F,Token:UUID From cb7ae0f08335e80fe16160f5fde3966129773eab Mon Sep 17 00:00:00 2001 From: Roman Babenko Date: Sun, 8 Sep 2024 13:40:33 +0300 Subject: [PATCH 49/49] Skip XML --- .ci/benchmark.txt | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/.ci/benchmark.txt b/.ci/benchmark.txt index 716dcf46..9eee3184 100644 --- a/.ci/benchmark.txt +++ b/.ci/benchmark.txt @@ -1,6 +1,6 @@ -META MD5 a384744d4dba5026f787a85f77a77b62 -DATA MD5 c562bf518eda436ca5a530c176aa0338 -DATA: 16345328 interested lines. MARKUP: 62823 items +META MD5 95bae3c3ad425d8f2073a121701a71d4 +DATA MD5 4ecb4c9436742b6b281ce7977066099d +DATA: 16344639 interested lines. MARKUP: 62814 items FileType FileNumber ValidLines Positives Negatives Templates --------------- ------------ ------------ ----------- ----------- ----------- 194 28318 71 418 90 @@ -217,13 +217,12 @@ FileType FileNumber ValidLines Positives Negatives Templat .xaml 21 8103 163 .xcscheme 1 109 6 .xib 11 503 169 -.xml 9 689 9 .xsl 1 311 1 .yaml 137 19004 125 345 42 .yml 419 36169 558 887 377 .zsh 6 872 12 .zsh-theme 1 97 1 -TOTAL: 10263 16345328 12204 50513 5107 +TOTAL: 10254 16344639 12204 50504 5107 credsweeper result_cnt : 0, lost_cnt : 0, true_cnt : 0, false_cnt : 0 Rules Positives Negatives Templates Reported TP FP TN FN FPR FNR ACC PRC RCL F1 ------------------------------ ----------- ----------- ----------- ---------- ---- ---- ----- ----- -------- -------- -------- ----- -------- ---- @@ -243,7 +242,7 @@ CMD Password 21 128 6 CMD Secret 1 1 0 0 0 1 1 0.000000 1.000000 0.500000 0.000000 CMD Token 6 0 0 0 0 0 6 1.000000 0.000000 0.000000 Certificate 24 471 0 0 0 471 24 0.000000 1.000000 0.951515 0.000000 -Credential 93 420 76 0 0 496 93 0.000000 1.000000 0.842105 0.000000 +Credential 93 419 76 0 0 495 93 0.000000 1.000000 0.841837 0.000000 Docker Swarm Token 2 0 0 0 0 0 2 1.000000 0.000000 0.000000 Dropbox App secret 64 139 1 0 0 140 64 0.000000 1.000000 0.686275 0.000000 Facebook Access Token 0 1 0 0 0 1 0 0.000000 1.000000 @@ -262,7 +261,7 @@ Key 3906 15720 485 Nonce 91 49 0 0 0 49 91 0.000000 1.000000 0.350000 0.000000 Other 8 8292 1 0 0 8293 8 0.000000 1.000000 0.999036 0.000000 PEM Private Key 1019 1483 0 0 0 1483 1019 0.000000 1.000000 0.592726 0.000000 -Password 1862 7539 2683 0 0 10222 1862 0.000000 1.000000 0.845912 0.000000 +Password 1862 7531 2683 0 0 10214 1862 0.000000 1.000000 0.845810 0.000000 Salt 47 76 1 0 0 77 47 0.000000 1.000000 0.620968 0.000000 Secret 1297 1576 802 0 0 2378 1297 0.000000 1.000000 0.647075 0.000000 Seed 1 6 0 0 0 6 1 0.000000 1.000000 0.857143 0.000000 @@ -271,4 +270,4 @@ Token 643 4168 454 Twilio API Key 0 5 2 0 0 7 0 0.000000 1.000000 URL Credentials 210 156 216 0 0 372 210 0.000000 1.000000 0.639175 0.000000 UUID 1069 265 0 0 0 265 1069 0.000000 1.000000 0.198651 0.000000 - 12204 50513 5107 0 0 0 50513 12204 0.000000 1.000000 0.805412 0.000000 + 12204 50504 5107 0 0 0 50504 12204 0.000000 1.000000 0.805384 0.000000