commit 2290c5881ff302dc009ac741b92a8682a39c1a38
parent 4907828757ec9a80c6358bf6b60ffebc15fca17f
Author: MTRNord <mtrnord1@gmail.com>
Date: Sun, 25 Feb 2024 12:30:40 +0100
Fix hashes
Diffstat:
8 files changed, 60 insertions(+), 34 deletions(-)
diff --git a/cmd/scanner.go b/cmd/scanner.go
@@ -5,6 +5,7 @@ import (
"log"
"os"
"path/filepath"
+ "runtime/pprof"
pdq "github.com/MTRNord/pdqhash-go"
"github.com/MTRNord/pdqhash-go/types"
@@ -14,7 +15,7 @@ import (
// This is an example. It is not meant to be run in prod.
-func processFile(filename string, detailed bool) {
+func processFolder(filename string, detailed bool) {
pdqhasher := pdq.NewPDQHasher()
numPDQHash := 0
@@ -49,23 +50,41 @@ func processFile(filename string, detailed bool) {
})
}
+func processFile(filename string, detailed bool) {
+ pdqhasher := pdq.NewPDQHasher()
+
+ hashAndQuality := pdqhasher.FromFile(filename)
+ delta := 0
+ if detailed {
+ log.Printf("hash=%s,norm=%d,delta=%d,quality=%d,filename=%s", hashAndQuality.Hash.String(), hashAndQuality.Hash.HammingNorm(), delta, hashAndQuality.Quality, filename)
+ } else {
+ log.Printf("%s,%d,%s", hashAndQuality.Hash.String(), hashAndQuality.Quality, filename)
+ }
+}
+
func main() {
var folder string
var detailedOutput bool
flag.StringVar(&folder, "folder", "", "Folder to scan")
flag.BoolVar(&detailedOutput, "detailed", false, "Detailed output")
+ var cpuprofile = flag.String("cpuprofile", "", "write cpu profile to file")
flag.Parse()
+ if *cpuprofile != "" {
+ f, err := os.Create(*cpuprofile)
+ if err != nil {
+ log.Fatal(err)
+ }
+ pprof.StartCPUProfile(f)
+ defer pprof.StopCPUProfile()
+ }
// Check if folder exists and is a folder
fileInfo, err := os.Stat(folder)
if err != nil {
log.Fatal(err)
}
- if !fileInfo.IsDir() {
- log.Fatalf("'%s' is not a folder", folder)
- }
vips.LoggingSettings(nil, vips.LogLevelMessage)
vips.Startup(&vips.Config{
@@ -78,6 +97,9 @@ func main() {
CollectStats: false,
})
defer vips.Shutdown()
-
- processFile(folder, detailedOutput)
+ if !fileInfo.IsDir() {
+ processFile(folder, detailedOutput)
+ } else {
+ processFolder(folder, detailedOutput)
+ }
}
diff --git a/go.mod b/go.mod
@@ -10,6 +10,7 @@ require (
)
require (
+ github.com/chewxy/math32 v1.10.1
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/stretchr/objx v0.5.0 // indirect
diff --git a/go.sum b/go.sum
@@ -1,3 +1,5 @@
+github.com/chewxy/math32 v1.10.1 h1:LFpeY0SLJXeaiej/eIp2L40VYfscTvKh/FSEZ68uMkU=
+github.com/chewxy/math32 v1.10.1/go.mod h1:dOB2rcuFrCn6UHrze36WSLVPKtzPMRAQvBvUwkSsLqs=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
diff --git a/helpers/helpers.go b/helpers/helpers.go
@@ -17,9 +17,9 @@ func Torben(m [][]float64, numRows, numCols int) float64 {
equal := 0
min := m[0][0]
max := m[0][0]
- guess := 0.0
- maxltguess := 0.0
- mingtguess := 0.0
+ guess := float64(0.0)
+ maxltguess := float64(0.0)
+ mingtguess := float64(0.0)
for i := 0; i < numRows; i++ {
for j := 0; j < numCols; j++ {
v := m[i][j]
diff --git a/helpers/helpers_test.go b/helpers/helpers_test.go
@@ -17,5 +17,5 @@ func TestTorben(t *testing.T) {
}
}
- assert.Equal(t, 1.07, Torben(m, numRows, numCols), "The Torben function should produce 1.07 for 3 rows and 8 cols")
+ assert.Equal(t, float64(1.07), Torben(m, numRows, numCols), "The Torben function should produce 1.07 for 3 rows and 8 cols")
}
diff --git a/pdq_hasher.go b/pdq_hasher.go
@@ -4,6 +4,7 @@ package pdq
import (
"log"
+
"math"
"github.com/MTRNord/pdqhash-go/helpers"
@@ -11,7 +12,6 @@ import (
"github.com/davidbyttow/govips/v2/vips"
_ "image/jpeg"
- _ "image/png"
)
// From Wikipedia: standard RGB to luminance (the 'Y' in 'YUV').
@@ -88,9 +88,9 @@ func NewPDQHasher() *PDQHasher {
func ComputeDCTMatrix() [][]float64 {
d := make([][]float64, 16)
- for i := 0; i < 16; i++ {
+ for i := 0; i < len(d); i++ {
di := make([]float64, 64)
- for j := 0; j < 64; j++ {
+ for j := 0; j < len(di); j++ {
di[j] = DCT_MATRIX_SCALE_FACTOR() * math.Cos((math.Pi/2.0/64.0)*(float64(i)+1.0)*(2.0*float64(j)+1.0))
}
d[i] = di
@@ -121,7 +121,7 @@ func (p *PDQHasher) FromFile(filename string) HashAndQuality {
}
// resizing the image proportionally to max 512px width and max 512px height
- err = image.Thumbnail(512, 512, vips.InterestingNone)
+ err = image.ThumbnailWithSize(512, 512, vips.InterestingNone, vips.SizeDown)
if err != nil {
log.Fatalf("Error resizing image: %v", err)
}
@@ -154,15 +154,17 @@ func (p *PDQHasher) fillFloatLumaFromBufferImage(image *vips.ImageRef, luma *[]f
if err != nil {
log.Fatalf("Error converting to RGB: %v", err)
}
- goImage, err := image.ToImage(nil)
- if err != nil {
- log.Fatalf("Error converting to Go image: %v", err)
- }
for i := 0; i < numRows; i++ {
for j := 0; j < numCols; j++ {
- colorArray := goImage.At(j, i)
- r, g, b, _ := colorArray.RGBA()
+ colorArray, err := image.GetPoint(j, i)
+ if err != nil {
+ log.Fatalf("Error getting pixel: %v", err)
+ }
+ r := colorArray[0]
+ g := colorArray[1]
+ b := colorArray[2]
+ log.Println(r, g, b)
(*luma)[i*numCols+j] = LUMA_FROM_R_COEFF*float64(r) + LUMA_FROM_G_COEFF*float64(g) + LUMA_FROM_B_COEFF*float64(b)
}
}
@@ -273,9 +275,9 @@ func (p *PDQHasher) pdqHash256esFromFloatLuma(fullBuffer1, fullBuffer2 []float64
// numRows x numCols in row-major order
func (p *PDQHasher) decimateFloat(in *[]float64, inNumRows, inNumCols int, out *[][]float64) {
for i := 0; i < 64; i++ {
- ini := int(((float64(i) + 0.5) * float64(inNumRows)) / 64)
+ ini := int(((float64(i) + 0.5) * float64(inNumRows)) / 64.0)
for j := 0; j < 64; j++ {
- inj := int(((float64(j) + 0.5) * float64(inNumCols)) / 64)
+ inj := int(((float64(j) + 0.5) * float64(inNumCols)) / 64.0)
(*out)[i][j] = (*in)[ini*inNumCols+inj]
}
}
@@ -293,7 +295,7 @@ func (p *PDQHasher) computePDQImageDomainQualityMetric(buffer64x64 [][]float64)
for j := 0; j < 64; j++ {
u := buffer64x64[i][j]
v := buffer64x64[i+1][j]
- d := int(((u - v) * 100) / 255)
+ d := int(((u - v) * 100.0) / 255.0)
gradientSum += int(helpers.Abs(d))
}
}
@@ -301,15 +303,15 @@ func (p *PDQHasher) computePDQImageDomainQualityMetric(buffer64x64 [][]float64)
for j := 0; j < 63; j++ {
u := buffer64x64[i][j]
v := buffer64x64[i][j+1]
- d := int(((u - v) * 100) / 255)
+ d := int(((u - v) * 100.0) / 255.0)
gradientSum += int(helpers.Abs(d))
}
}
- quality := int(gradientSum / 90)
+ quality := float64(gradientSum) / 90.0
if quality > 100 {
quality = 100
}
- return quality
+ return int(quality)
}
/**
@@ -329,7 +331,7 @@ func (p *PDQHasher) dct64To16(A, T, B *([][]float64)) {
ti := make([]float64, 64)
for j := 0; j < 64; j++ {
- tij := 0.0
+ tij := float64(0.0)
for k := 0; k < 64; k++ {
tij += D[i][k] * (*A)[k][j]
}
@@ -340,7 +342,7 @@ func (p *PDQHasher) dct64To16(A, T, B *([][]float64)) {
for i := 0; i < 16; i++ {
for j := 0; j < 16; j++ {
- sumk := 0.0
+ sumk := float64(0.0)
for k := 0; k < 64; k++ {
sumk += (*T)[i][k] * D[j][k]
}
@@ -466,7 +468,8 @@ func (p *PDQHasher) pdqBuffer16x16ToBits(dctOutput16x16 [][]float64) *types.Hash
// Round up.
func (p *PDQHasher) computeJaroszWindowSize(dimension int) int {
- return (dimension + PDQ_JAROSZ_WINDOW_SIZE_DIVISOR - 1) / PDQ_JAROSZ_WINDOW_SIZE_DIVISOR
+ result := (float64(dimension) + float64(PDQ_JAROSZ_WINDOW_SIZE_DIVISOR) - 1.0) / float64(PDQ_JAROSZ_WINDOW_SIZE_DIVISOR)
+ return int(result)
}
func (p *PDQHasher) jaroszFilterFloat(buffer1, buffer2 *[]float64, numRows, numCols, windowSizeAlongRows, windowSizeAlongCols, nreps int) {
@@ -477,7 +480,7 @@ func (p *PDQHasher) jaroszFilterFloat(buffer1, buffer2 *[]float64, numRows, numC
}
func (p *PDQHasher) box1DFloat(invec *[]float64, inStartOffset int, outvec *[]float64, outStartOffset, vectorLength, stride, fullWindowSize int) {
- halfWindowSize := int((fullWindowSize + 2) / 2)
+ halfWindowSize := int(((float64(fullWindowSize) + 2.0) / 2.0))
phase_1_nreps := int(halfWindowSize - 1)
phase_2_nreps := int(fullWindowSize - halfWindowSize + 1)
phase_3_nreps := int(vectorLength - fullWindowSize)
@@ -485,7 +488,7 @@ func (p *PDQHasher) box1DFloat(invec *[]float64, inStartOffset int, outvec *[]fl
li := 0 // Index of left edge of read window, for subtracts
ri := 0 // Index of right edge of read windows, for adds
oi := 0 // Index of output vector
- sum := 0.0
+ sum := float64(0.0)
currentWindowSize := 0
// PHASE 1: ACCUMULATE FIRST SUM NO WRITES
diff --git a/pdq_hasher_test.go b/pdq_hasher_test.go
@@ -1,7 +1,6 @@
package pdq
import (
- "log"
"testing"
"github.com/MTRNord/pdqhash-go/types"
@@ -120,7 +119,6 @@ func TestPDQHasher(t *testing.T) {
computedHash := hash.Hash
hammingDistance := computedHash.HammingDistance(expectedHashObj)
- log.Printf("Image: %s Hamming distance: %d\n", imagePath, hammingDistance)
- assert.True(t, hammingDistance <= hammingTolerance)
+ assert.LessOrEqualf(t, hammingDistance, hammingTolerance, "Hamming distance is too high: %s", imagePath)
}
}
diff --git a/q0004.prof b/q0004.prof
Binary files differ.
