ILFBB4ISBFKXETYLV7QEO4SKOMPHSWRLKN3MVHEGFOPQ5RQJWDDQC 7CC2YVZXAIUNWXNNVIO5KOZZFDQQLESFO72SGEDP2C4OZXAWO4KQC VT3A2ORJW3CJV4VZEKVQEU6XZ35RNBLMSQJOKYHATXKGMZLAZKQAC IFVRAERTCCDICNTYTG3TX2WASB6RXQQEJWWXQMQZJSQDQ3HLE5OQC D4EL6RSTSZ3S3IDSETRNGLJHZKGZEE2V2OZIOKQK6LRLHQNS77JQC 3JA7HYRMHV57SIMGMGPDOMKQ3NBQS2SKOX3EKDHRBQRP7ZPZGFTQC RFSUR7ZEXTQNHH3IFJAL2NNOTGRPWOWB3PFIVH7VLI2JPTIBMW5AC U6JEEU5O477ZOJ5UMRMOJSGPEJEU6Q7KMPKUSDF56CYVUJWL7QBQC W3A2EECCD23SVHJZN6MXPH2PAVFHH5CNFD2XHPQRRW6M4GUTG3FAC package toolsimport ("bufio""fmt""os""path/filepath""sort""strings""skraak/utils")// CallsFromRavenInput defines the input for the calls-from-raven tooltype CallsFromRavenInput struct {Folder string `json:"folder" jsonschema:"Folder containing Raven selection files"`File string `json:"file" jsonschema:"Single Raven selection file to process"`Delete bool `json:"delete" jsonschema:"Delete Raven files after processing"`ProgressHandler ProgressHandler `json:"-"` // Optional progress callback}// CallsFromRavenOutput defines the output for the calls-from-raven tooltype CallsFromRavenOutput struct {Calls []ClusteredCall `json:"calls"`TotalCalls int `json:"total_calls"`SpeciesCount map[string]int `json:"species_count"`DataFilesWritten int `json:"data_files_written"`DataFilesSkipped int `json:"data_files_skipped"`FilesProcessed int `json:"files_processed"`FilesDeleted int `json:"files_deleted"`Filter string `json:"filter"`Error *string `json:"error,omitempty"`}// RavenSelection represents a single Raven selectiontype RavenSelection struct {StartTime float64EndTime float64FreqLow float64FreqHigh float64Species string}// CallsFromRaven processes Raven selection files and writes .data filesfunc CallsFromRaven(input CallsFromRavenInput) (CallsFromRavenOutput, error) {var output CallsFromRavenOutputoutput.Filter = "Raven"// Collect Raven files to processvar ravenFiles []stringif input.File != "" {ravenFiles = []string{input.File}} else if input.Folder != "" {var err errorravenFiles, err = findRavenFiles(input.Folder)if err != nil {errMsg := fmt.Sprintf("Failed to find Raven files: %v", err)output.Error = &errMsgreturn output, fmt.Errorf("%s", errMsg)}} else {errMsg := "Either --folder or --file must be specified"output.Error = &errMsgreturn output, fmt.Errorf("%s", errMsg)}if len(ravenFiles) == 0 {errMsg := "No Raven files found"output.Error = &errMsgreturn output, fmt.Errorf("%s", errMsg)}// Process each Raven filespeciesCount := make(map[string]int)var allCalls []ClusteredCalldataFilesWritten := 0dataFilesSkipped := 0filesProcessed := 0filesDeleted := 0for _, ravenFile := range ravenFiles {calls, written, skipped, err := processRavenFile(ravenFile)if err != nil {// Stop on first errorerrMsg := fmt.Sprintf("Error processing %s: %v", ravenFile, err)output.Error = &errMsgreturn output, fmt.Errorf("%s", errMsg)}if written {dataFilesWritten++}if skipped {dataFilesSkipped++}for _, call := range calls {allCalls = append(allCalls, call)speciesCount[call.EbirdCode]++}filesProcessed++// Delete if requested and successfully processedif input.Delete && written {if err := os.Remove(ravenFile); err != nil {errMsg := fmt.Sprintf("Failed to delete %s: %v", ravenFile, err)output.Error = &errMsgreturn output, fmt.Errorf("%s", errMsg)}filesDeleted++}if input.ProgressHandler != nil {input.ProgressHandler(filesProcessed, len(ravenFiles), filepath.Base(ravenFile))}}// Sort all calls by file, then start timesort.Slice(allCalls, func(i, j int) bool {if allCalls[i].File != allCalls[j].File {return allCalls[i].File < allCalls[j].File}return allCalls[i].StartTime < allCalls[j].StartTime})output.Calls = allCallsoutput.TotalCalls = len(allCalls)output.SpeciesCount = speciesCountoutput.DataFilesWritten = dataFilesWrittenoutput.DataFilesSkipped = dataFilesSkippedoutput.FilesProcessed = filesProcessedoutput.FilesDeleted = filesDeletedreturn output, nil}// findRavenFiles finds all Raven selection files in a folderfunc findRavenFiles(folder string) ([]string, error) {var files []stringentries, err := os.ReadDir(folder)if err != nil {return nil, err}for _, entry := range entries {name := entry.Name()if strings.HasSuffix(name, ".selections.txt") {files = append(files, filepath.Join(folder, name))}}return files, nil}// processRavenFile processes a single Raven selection filefunc processRavenFile(ravenFile string) ([]ClusteredCall, bool, bool, error) {// Open filefile, err := os.Open(ravenFile)if err != nil {return nil, false, false, fmt.Errorf("failed to open file: %w", err)}defer file.Close()// Read header and selections (tab-separated)scanner := bufio.NewScanner(file)// Read header lineif !scanner.Scan() {return nil, false, false, fmt.Errorf("empty file")}header := strings.Split(scanner.Text(), "\t")// Find column indicesbeginTimeIdx := -1endTimeIdx := -1lowFreqIdx := -1highFreqIdx := -1speciesIdx := -1for i, col := range header {switch col {case "Begin Time (s)":beginTimeIdx = icase "End Time (s)":endTimeIdx = icase "Low Freq (Hz)":lowFreqIdx = icase "High Freq (Hz)":highFreqIdx = icase "Species":speciesIdx = i}}if beginTimeIdx == -1 || endTimeIdx == -1 || speciesIdx == -1 {return nil, false, false, fmt.Errorf("missing required columns in Raven file")}// Read selectionsvar selections []RavenSelectionfor scanner.Scan() {line := scanner.Text()if line == "" {continue}fields := strings.Split(line, "\t")if len(fields) <= speciesIdx {continue}var sel RavenSelectionfmt.Sscanf(fields[beginTimeIdx], "%f", &sel.StartTime)fmt.Sscanf(fields[endTimeIdx], "%f", &sel.EndTime)if lowFreqIdx >= 0 && lowFreqIdx < len(fields) {fmt.Sscanf(fields[lowFreqIdx], "%f", &sel.FreqLow)}if highFreqIdx >= 0 && highFreqIdx < len(fields) {fmt.Sscanf(fields[highFreqIdx], "%f", &sel.FreqHigh)}sel.Species = fields[speciesIdx]selections = append(selections, sel)}if err := scanner.Err(); err != nil {return nil, false, false, fmt.Errorf("error reading file: %w", err)}if len(selections) == 0 {return nil, false, true, nil // No selections, skip}// Derive WAV path from Raven filename// "20230610_150000.Table.1.selections.txt" -> "20230610_150000.WAV"base := filepath.Base(ravenFile)// Remove .selections.txtnameWithoutSuffix := strings.TrimSuffix(base, ".selections.txt")// Remove .Table.X (or similar pattern)idx := strings.Index(nameWithoutSuffix, ".Table.")if idx > 0 {nameWithoutSuffix = nameWithoutSuffix[:idx]}wavBase := nameWithoutSuffix + ".WAV"wavPath := filepath.Join(filepath.Dir(ravenFile), wavBase)// Check if WAV exists (to get sample rate and duration)sampleRate, duration, err := utils.ParseWAVHeaderMinimal(wavPath)if err != nil {return nil, false, true, nil // Skip if WAV not found or invalid}dataPath := wavPath + ".data"// Convert selections to segmentssegments := buildRavenSegments(selections, sampleRate)// Build metadatameta := AviaNZMeta{Operator: "Raven",Duration: duration,}reviewer := "None"meta.Reviewer = &reviewer// Write .data file (safe write)if err := writeDotDataFileSafe(dataPath, segments, "Raven", meta); err != nil {return nil, false, false, err}// Convert to ClusteredCalls for outputvar calls []ClusteredCallfor _, sel := range selections {calls = append(calls, ClusteredCall{File: wavPath,StartTime: sel.StartTime,EndTime: sel.EndTime,EbirdCode: sel.Species,Segments: 1,})}return calls, true, false, nil}// buildRavenSegments converts Raven selections to AviaNZ segmentsfunc buildRavenSegments(selections []RavenSelection, sampleRate int) []AviaNZSegment {var segments []AviaNZSegmentfor _, sel := range selections {labels := []AviaNZLabel{{Species: sel.Species,Certainty: 70, // Default certainty for Raven (no confidence metric)Filter: "Raven",},}// Use frequency range from Raven, or full band if not specifiedfreqLow := sel.FreqLowfreqHigh := sel.FreqHighif freqLow == 0 && freqHigh == 0 {freqHigh = float64(sampleRate)}segment := AviaNZSegment{sel.StartTime,sel.EndTime,freqLow,freqHigh,labels,}segments = append(segments, segment)}return segments}
package toolsimport ("os""path/filepath""testing""skraak/utils")// ============================================// BirdNET Tests// ============================================func TestCallsFromBirda_NewDataFile(t *testing.T) {tmpDir := t.TempDir()// Create a minimal WAV filewavPath := filepath.Join(tmpDir, "test.WAV")createMinimalWAV(t, wavPath, 16000, 60.0)// Create BirdNET results filebirdaPath := filepath.Join(tmpDir, "test.BirdNET.results.csv")birdaContent := "\ufeffStart (s),End (s),Scientific name,Common name,Confidence,File\n0.0,3.0,Turdus migratorius,American Robin,0.85,/some/path/test.WAV\n"if err := os.WriteFile(birdaPath, []byte(birdaContent), 0644); err != nil {t.Fatal(err)}input := CallsFromBirdaInput{File: birdaPath,}output, err := CallsFromBirda(input)if err != nil {t.Fatalf("unexpected error: %v", err)}if output.DataFilesWritten != 1 {t.Errorf("expected 1 data file written, got %d", output.DataFilesWritten)}if output.Filter != "BirdNET" {t.Errorf("expected filter 'BirdNET', got '%s'", output.Filter)}if output.TotalCalls != 1 {t.Errorf("expected 1 call, got %d", output.TotalCalls)}// Verify .data file was createddataPath := wavPath + ".data"df, err := utils.ParseDataFile(dataPath)if err != nil {t.Fatalf("failed to parse .data file: %v", err)}if len(df.Segments) != 1 {t.Errorf("expected 1 segment, got %d", len(df.Segments))}if df.Segments[0].Labels[0].Filter != "BirdNET" {t.Errorf("expected filter 'BirdNET', got '%s'", df.Segments[0].Labels[0].Filter)}if df.Segments[0].Labels[0].Certainty != 85 {t.Errorf("expected certainty 85, got %d", df.Segments[0].Labels[0].Certainty)}}func TestCallsFromBirda_ExistingSameFilter(t *testing.T) {tmpDir := t.TempDir()wavPath := filepath.Join(tmpDir, "test.WAV")createMinimalWAV(t, wavPath, 16000, 60.0)dataPath := wavPath + ".data"existingData := `[{"Operator": "Test", "Duration": 60.0}, [5.0, 10.0, 0, 16000, [{"species": "Existing Bird", "certainty": 90, "filter": "BirdNET"}]]]`if err := os.WriteFile(dataPath, []byte(existingData), 0644); err != nil {t.Fatal(err)}birdaPath := filepath.Join(tmpDir, "test.BirdNET.results.csv")birdaContent := "\ufeffStart (s),End (s),Scientific name,Common name,Confidence,File\n0.0,3.0,New Bird,New Bird,0.85,test.WAV\n"if err := os.WriteFile(birdaPath, []byte(birdaContent), 0644); err != nil {t.Fatal(err)}input := CallsFromBirdaInput{File: birdaPath}output, err := CallsFromBirda(input)if err == nil {t.Error("expected error for same filter, got nil")}if output.Error == nil {t.Error("expected error message in output")}}func TestCallsFromBirda_ExistingDifferentFilter(t *testing.T) {tmpDir := t.TempDir()wavPath := filepath.Join(tmpDir, "test.WAV")createMinimalWAV(t, wavPath, 16000, 60.0)dataPath := wavPath + ".data"existingData := `[{"Operator": "Test", "Duration": 60.0}, [5.0, 10.0, 0, 16000, [{"species": "Kiwi", "certainty": 90, "filter": "Manual"}]]]`if err := os.WriteFile(dataPath, []byte(existingData), 0644); err != nil {t.Fatal(err)}birdaPath := filepath.Join(tmpDir, "test.BirdNET.results.csv")birdaContent := "\ufeffStart (s),End (s),Scientific name,Common name,Confidence,File\n0.0,3.0,Robin,Robin,0.85,test.WAV\n"if err := os.WriteFile(birdaPath, []byte(birdaContent), 0644); err != nil {t.Fatal(err)}input := CallsFromBirdaInput{File: birdaPath}output, err := CallsFromBirda(input)if err != nil {t.Fatalf("unexpected error: %v", err)}if output.DataFilesWritten != 1 {t.Errorf("expected 1 data file written, got %d", output.DataFilesWritten)}df, err := utils.ParseDataFile(dataPath)if err != nil {t.Fatalf("failed to parse .data file: %v", err)}if len(df.Segments) != 2 {t.Errorf("expected 2 segments after merge, got %d", len(df.Segments))}}func TestCallsFromBirda_DeleteOption(t *testing.T) {tmpDir := t.TempDir()wavPath := filepath.Join(tmpDir, "test.WAV")createMinimalWAV(t, wavPath, 16000, 60.0)birdaPath := filepath.Join(tmpDir, "test.BirdNET.results.csv")birdaContent := "\ufeffStart (s),End (s),Scientific name,Common name,Confidence,File\n0.0,3.0,Robin,Robin,0.85,test.WAV\n"if err := os.WriteFile(birdaPath, []byte(birdaContent), 0644); err != nil {t.Fatal(err)}input := CallsFromBirdaInput{File: birdaPath, Delete: true}output, err := CallsFromBirda(input)if err != nil {t.Fatalf("unexpected error: %v", err)}if output.FilesDeleted != 1 {t.Errorf("expected 1 file deleted, got %d", output.FilesDeleted)}if _, err := os.Stat(birdaPath); !os.IsNotExist(err) {t.Error("expected BirdNET file to be deleted")}}func TestCallsFromBirda_FolderMode(t *testing.T) {tmpDir := t.TempDir()for i := 0; i < 2; i++ {wavPath := filepath.Join(tmpDir, "test"+string(rune('0'+i))+".WAV")createMinimalWAV(t, wavPath, 16000, 60.0)birdaPath := filepath.Join(tmpDir, "test"+string(rune('0'+i))+".BirdNET.results.csv")birdaContent := "\ufeffStart (s),End (s),Scientific name,Common name,Confidence,File\n0.0,3.0,Bird,Bird,0.85,test.WAV\n"if err := os.WriteFile(birdaPath, []byte(birdaContent), 0644); err != nil {t.Fatal(err)}}input := CallsFromBirdaInput{Folder: tmpDir}output, err := CallsFromBirda(input)if err != nil {t.Fatalf("unexpected error: %v", err)}if output.FilesProcessed != 2 {t.Errorf("expected 2 files processed, got %d", output.FilesProcessed)}if output.DataFilesWritten != 2 {t.Errorf("expected 2 data files written, got %d", output.DataFilesWritten)}}// ============================================// Raven Tests// ============================================func TestCallsFromRaven_NewDataFile(t *testing.T) {tmpDir := t.TempDir()wavPath := filepath.Join(tmpDir, "test.WAV")createMinimalWAV(t, wavPath, 16000, 60.0)ravenPath := filepath.Join(tmpDir, "test.Table.1.selections.txt")ravenContent := "Selection\tView\tChannel\tBegin Time (s)\tEnd Time (s)\tLow Freq (Hz)\tHigh Freq (Hz)\tSpecies\n1\tSpectrogram 1\t1\t0.0\t5.0\t1000\t5000\tKiwi\n"if err := os.WriteFile(ravenPath, []byte(ravenContent), 0644); err != nil {t.Fatal(err)}input := CallsFromRavenInput{File: ravenPath}output, err := CallsFromRaven(input)if err != nil {t.Fatalf("unexpected error: %v", err)}if output.DataFilesWritten != 1 {t.Errorf("expected 1 data file written, got %d", output.DataFilesWritten)}if output.Filter != "Raven" {t.Errorf("expected filter 'Raven', got '%s'", output.Filter)}dataPath := wavPath + ".data"df, err := utils.ParseDataFile(dataPath)if err != nil {t.Fatalf("failed to parse .data file: %v", err)}if df.Segments[0].FreqLow != 1000 {t.Errorf("expected freq_low 1000, got %f", df.Segments[0].FreqLow)}if df.Segments[0].FreqHigh != 5000 {t.Errorf("expected freq_high 5000, got %f", df.Segments[0].FreqHigh)}}func TestCallsFromRaven_ExistingSameFilter(t *testing.T) {tmpDir := t.TempDir()wavPath := filepath.Join(tmpDir, "test.WAV")createMinimalWAV(t, wavPath, 16000, 60.0)dataPath := wavPath + ".data"existingData := `[{"Operator": "Test", "Duration": 60.0}, [5.0, 10.0, 0, 16000, [{"species": "Existing", "certainty": 90, "filter": "Raven"}]]]`if err := os.WriteFile(dataPath, []byte(existingData), 0644); err != nil {t.Fatal(err)}ravenPath := filepath.Join(tmpDir, "test.Table.1.selections.txt")ravenContent := "Selection\tView\tChannel\tBegin Time (s)\tEnd Time (s)\tLow Freq (Hz)\tHigh Freq (Hz)\tSpecies\n1\tSpectrogram 1\t1\t0.0\t5.0\t1000\t5000\tNew\n"if err := os.WriteFile(ravenPath, []byte(ravenContent), 0644); err != nil {t.Fatal(err)}input := CallsFromRavenInput{File: ravenPath}output, err := CallsFromRaven(input)if err == nil {t.Error("expected error for same filter, got nil")}if output.Error == nil {t.Error("expected error message in output")}}func TestCallsFromRaven_ExistingDifferentFilter(t *testing.T) {tmpDir := t.TempDir()wavPath := filepath.Join(tmpDir, "test.WAV")createMinimalWAV(t, wavPath, 16000, 60.0)dataPath := wavPath + ".data"existingData := `[{"Operator": "Test", "Duration": 60.0}, [5.0, 10.0, 0, 16000, [{"species": "Kiwi", "certainty": 90, "filter": "BirdNET"}]]]`if err := os.WriteFile(dataPath, []byte(existingData), 0644); err != nil {t.Fatal(err)}ravenPath := filepath.Join(tmpDir, "test.Table.1.selections.txt")ravenContent := "Selection\tView\tChannel\tBegin Time (s)\tEnd Time (s)\tLow Freq (Hz)\tHigh Freq (Hz)\tSpecies\n1\tSpectrogram 1\t1\t0.0\t5.0\t1000\t5000\tMorepork\n"if err := os.WriteFile(ravenPath, []byte(ravenContent), 0644); err != nil {t.Fatal(err)}input := CallsFromRavenInput{File: ravenPath}output, err := CallsFromRaven(input)if err != nil {t.Fatalf("unexpected error: %v", err)}if output.DataFilesWritten != 1 {t.Errorf("expected 1 data file written, got %d", output.DataFilesWritten)}df, err := utils.ParseDataFile(dataPath)if err != nil {t.Fatalf("failed to parse .data file: %v", err)}if len(df.Segments) != 2 {t.Errorf("expected 2 segments after merge, got %d", len(df.Segments))}}func TestCallsFromRaven_DeleteOption(t *testing.T) {tmpDir := t.TempDir()wavPath := filepath.Join(tmpDir, "test.WAV")createMinimalWAV(t, wavPath, 16000, 60.0)ravenPath := filepath.Join(tmpDir, "test.Table.1.selections.txt")ravenContent := "Selection\tView\tChannel\tBegin Time (s)\tEnd Time (s)\tLow Freq (Hz)\tHigh Freq (Hz)\tSpecies\n1\tSpectrogram 1\t1\t0.0\t5.0\t1000\t5000\tKiwi\n"if err := os.WriteFile(ravenPath, []byte(ravenContent), 0644); err != nil {t.Fatal(err)}input := CallsFromRavenInput{File: ravenPath, Delete: true}output, err := CallsFromRaven(input)if err != nil {t.Fatalf("unexpected error: %v", err)}if output.FilesDeleted != 1 {t.Errorf("expected 1 file deleted, got %d", output.FilesDeleted)}if _, err := os.Stat(ravenPath); !os.IsNotExist(err) {t.Error("expected Raven file to be deleted")}}func TestCallsFromRaven_MultipleSelections(t *testing.T) {tmpDir := t.TempDir()wavPath := filepath.Join(tmpDir, "test.WAV")createMinimalWAV(t, wavPath, 16000, 60.0)ravenPath := filepath.Join(tmpDir, "test.Table.1.selections.txt")ravenContent := "Selection\tView\tChannel\tBegin Time (s)\tEnd Time (s)\tLow Freq (Hz)\tHigh Freq (Hz)\tSpecies\n1\tSpectrogram 1\t1\t0.0\t5.0\t1000\t5000\tKiwi\n2\tSpectrogram 1\t1\t10.0\t15.0\t2000\t6000\tMorepork\n3\tSpectrogram 1\t1\t20.0\t25.0\t1500\t4500\tTui\n"if err := os.WriteFile(ravenPath, []byte(ravenContent), 0644); err != nil {t.Fatal(err)}input := CallsFromRavenInput{File: ravenPath}output, err := CallsFromRaven(input)if err != nil {t.Fatalf("unexpected error: %v", err)}if output.TotalCalls != 3 {t.Errorf("expected 3 calls, got %d", output.TotalCalls)}if output.SpeciesCount["Kiwi"] != 1 || output.SpeciesCount["Morepork"] != 1 || output.SpeciesCount["Tui"] != 1 {t.Errorf("unexpected species count: %v", output.SpeciesCount)}}
package toolsimport ("encoding/csv""fmt""io""os""path/filepath""sort""strings""skraak/utils")// CallsFromBirdaInput defines the input for the calls-from-birda tooltype CallsFromBirdaInput struct {Folder string `json:"folder" jsonschema:"Folder containing BirdNET results files"`File string `json:"file" jsonschema:"Single BirdNET results file to process"`Delete bool `json:"delete" jsonschema:"Delete BirdNET files after processing"`ProgressHandler ProgressHandler `json:"-"` // Optional progress callback}// CallsFromBirdaOutput defines the output for the calls-from-birda tooltype CallsFromBirdaOutput struct {Calls []ClusteredCall `json:"calls"`TotalCalls int `json:"total_calls"`SpeciesCount map[string]int `json:"species_count"`DataFilesWritten int `json:"data_files_written"`DataFilesSkipped int `json:"data_files_skipped"`FilesProcessed int `json:"files_processed"`FilesDeleted int `json:"files_deleted"`Filter string `json:"filter"`Error *string `json:"error,omitempty"`}// BirdNETDetection represents a single BirdNET detectiontype BirdNETDetection struct {StartTime float64EndTime float64ScientificName stringCommonName stringConfidence float64WAVPath string}// CallsFromBirda processes BirdNET results files and writes .data filesfunc CallsFromBirda(input CallsFromBirdaInput) (CallsFromBirdaOutput, error) {var output CallsFromBirdaOutputoutput.Filter = "BirdNET"// Collect BirdNET files to processvar birdaFiles []stringif input.File != "" {birdaFiles = []string{input.File}} else if input.Folder != "" {var err errorbirdaFiles, err = findBirdaFiles(input.Folder)if err != nil {errMsg := fmt.Sprintf("Failed to find BirdNET files: %v", err)output.Error = &errMsgreturn output, fmt.Errorf("%s", errMsg)}} else {errMsg := "Either --folder or --file must be specified"output.Error = &errMsgreturn output, fmt.Errorf("%s", errMsg)}if len(birdaFiles) == 0 {errMsg := "No BirdNET files found"output.Error = &errMsgreturn output, fmt.Errorf("%s", errMsg)}// Process each BirdNET filespeciesCount := make(map[string]int)var allCalls []ClusteredCalldataFilesWritten := 0dataFilesSkipped := 0filesProcessed := 0filesDeleted := 0for _, birdaFile := range birdaFiles {calls, written, skipped, err := processBirdaFile(birdaFile)if err != nil {// Stop on first errorerrMsg := fmt.Sprintf("Error processing %s: %v", birdaFile, err)output.Error = &errMsgreturn output, fmt.Errorf("%s", errMsg)}if written {dataFilesWritten++}if skipped {dataFilesSkipped++}for _, call := range calls {allCalls = append(allCalls, call)speciesCount[call.EbirdCode]++}filesProcessed++// Delete if requested and successfully processedif input.Delete && written {if err := os.Remove(birdaFile); err != nil {errMsg := fmt.Sprintf("Failed to delete %s: %v", birdaFile, err)output.Error = &errMsgreturn output, fmt.Errorf("%s", errMsg)}filesDeleted++}if input.ProgressHandler != nil {input.ProgressHandler(filesProcessed, len(birdaFiles), filepath.Base(birdaFile))}}// Sort all calls by file, then start timesort.Slice(allCalls, func(i, j int) bool {if allCalls[i].File != allCalls[j].File {return allCalls[i].File < allCalls[j].File}return allCalls[i].StartTime < allCalls[j].StartTime})output.Calls = allCallsoutput.TotalCalls = len(allCalls)output.SpeciesCount = speciesCountoutput.DataFilesWritten = dataFilesWrittenoutput.DataFilesSkipped = dataFilesSkippedoutput.FilesProcessed = filesProcessedoutput.FilesDeleted = filesDeletedreturn output, nil}// findBirdaFiles finds all BirdNET results files in a folderfunc findBirdaFiles(folder string) ([]string, error) {var files []stringentries, err := os.ReadDir(folder)if err != nil {return nil, err}for _, entry := range entries {name := entry.Name()if strings.HasSuffix(name, ".BirdNET.results.csv") {files = append(files, filepath.Join(folder, name))}}return files, nil}// processBirdaFile processes a single BirdNET results filefunc processBirdaFile(birdaFile string) ([]ClusteredCall, bool, bool, error) {// Open and parse CSVfile, err := os.Open(birdaFile)if err != nil {return nil, false, false, fmt.Errorf("failed to open file: %w", err)}defer file.Close()// Create CSV readerreader := csv.NewReader(file)// Read headerheader, err := reader.Read()if err != nil {return nil, false, false, fmt.Errorf("failed to read header: %w", err)}// Find column indices (handle BOM prefix)startIdx := -1endIdx := -1commonNameIdx := -1confidenceIdx := -1fileIdx := -1for i, col := range header {// Remove BOM if presentcol = strings.TrimPrefix(col, "\ufeff")switch col {case "Start (s)":startIdx = icase "End (s)":endIdx = icase "Common name":commonNameIdx = icase "Confidence":confidenceIdx = icase "File":fileIdx = i}}if startIdx == -1 || endIdx == -1 || commonNameIdx == -1 || confidenceIdx == -1 {return nil, false, false, fmt.Errorf("missing required columns in BirdNET file")}// Read detectionsvar detections []BirdNETDetectionfor {record, err := reader.Read()if err == io.EOF {break}if err != nil {return nil, false, false, fmt.Errorf("failed to read record: %w", err)}var det BirdNETDetectionfmt.Sscanf(record[startIdx], "%f", &det.StartTime)fmt.Sscanf(record[endIdx], "%f", &det.EndTime)det.CommonName = record[commonNameIdx]fmt.Sscanf(record[confidenceIdx], "%f", &det.Confidence)if fileIdx >= 0 && fileIdx < len(record) {det.WAVPath = record[fileIdx]}detections = append(detections, det)}if len(detections) == 0 {return nil, false, true, nil // No detections, skip}// Determine WAV path and .data pathvar wavPath stringif detections[0].WAVPath != "" {// Check if the path from File column existsif _, err := os.Stat(detections[0].WAVPath); err == nil {wavPath = detections[0].WAVPath} else {// Derive from BirdNET filename (File column may have wrong path)base := filepath.Base(birdaFile)wavBase := strings.TrimSuffix(base, ".BirdNET.results.csv") + ".WAV"wavPath = filepath.Join(filepath.Dir(birdaFile), wavBase)}} else {// Derive from BirdNET filenamebase := filepath.Base(birdaFile)wavBase := strings.TrimSuffix(base, ".BirdNET.results.csv") + ".WAV"wavPath = filepath.Join(filepath.Dir(birdaFile), wavBase)}// Check if WAV exists (to get sample rate and duration)sampleRate, duration, err := utils.ParseWAVHeaderMinimal(wavPath)if err != nil {return nil, false, true, nil // Skip if WAV not found or invalid}dataPath := wavPath + ".data"// Convert detections to segmentssegments := buildBirdNETSegments(detections, sampleRate)// Build metadatameta := AviaNZMeta{Operator: "BirdNET",Duration: duration,}reviewer := "None"meta.Reviewer = &reviewer// Write .data file (safe write)if err := writeDotDataFileSafe(dataPath, segments, "BirdNET", meta); err != nil {return nil, false, false, err}// Convert to ClusteredCalls for outputvar calls []ClusteredCallfor _, det := range detections {calls = append(calls, ClusteredCall{File: wavPath,StartTime: det.StartTime,EndTime: det.EndTime,EbirdCode: det.CommonName,Segments: 1,})}return calls, true, false, nil}// buildBirdNETSegments converts BirdNET detections to AviaNZ segmentsfunc buildBirdNETSegments(detections []BirdNETDetection, sampleRate int) []AviaNZSegment {var segments []AviaNZSegmentfor _, det := range detections {// Convert confidence (0.0-1.0) to certainty (0-100)certainty := int(det.Confidence * 100)if certainty < 0 {certainty = 0}if certainty > 100 {certainty = 100}labels := []AviaNZLabel{{Species: det.CommonName,Certainty: certainty,Filter: "BirdNET",},}segment := AviaNZSegment{det.StartTime,det.EndTime,0, // freq_lowsampleRate, // freq_high (full band)labels,}segments = append(segments, segment)}return segments}
PLAN MODE: Great, now that we have done that, lets implemehnt 2 new calls commands:calls from-birda --folder (or just 1 file with --file) --delete (optional, to delete the birda files as we go)calls from-raven --folder (or just 1 file with --file) --delete (optional, to delete the raven file as we go)We need to append birda and raven files to existing .data files, or create a new .data if not present.find sample birda files (1 or 0, per WAV) at /media/david/SSD4/Twenty_Four_Seven/R620/2024-05-06/20230610_150000.BirdNET.results.csv (note .BirdNET.results.csv) FILTER=BirdNETfind sample Raven files (1 or 0, per WAV) at /media/david/SSD4/Twenty_Four_Seven/R620/2024-05-06/20230610_150000.Table.1.selections.txt (note Table.$x.selections.txt, could be 1, 2, 3 ..., usually just 1, look for *.selections.txt) FILTER=RavenNote that both types of csv files are similar to .data files, they refer to a WAV file.Behaviour must be the same as calls from-preds, no clobber, we do not allow birda or raven files to be imported more than 1 timeWe do not allow manual overide for --filter, we always parse from the filename.filter BirdNET from *.BirdNET.results.csvfilter Raven from *.selections.txtboth new commands should output a json summary of calls parsed in the sae form as the output from calls from-preds cmdPLAN MODE
fmt.Fprintf(os.Stderr, " skraak calls from-preds --csv preds.csv --dot-data=false > calls.json\n")fmt.Fprintf(os.Stderr, " skraak calls from-preds --csv preds.csv --filter my-filter\n")
fmt.Fprintf(os.Stderr, " skraak calls from-birda --folder ./recordings\n")fmt.Fprintf(os.Stderr, " skraak calls from-raven --folder ./recordings --delete\n")
// runCallsFromBirda handles the "calls from-birda" subcommandfunc runCallsFromBirda(args []string) {fs := flag.NewFlagSet("calls from-birda", flag.ExitOnError)folder := fs.String("folder", "", "Folder containing BirdNET results files")file := fs.String("file", "", "Single BirdNET results file to process")delete := fs.Bool("delete", false, "Delete BirdNET files after processing")fs.Usage = func() {fmt.Fprintf(os.Stderr, "Usage: skraak calls from-birda [options]\n\n")fmt.Fprintf(os.Stderr, "Import BirdNET results to .data files.\n")fmt.Fprintf(os.Stderr, "Reads *.BirdNET.results.csv files and creates/merges .data files.\n\n")fmt.Fprintf(os.Stderr, "Options:\n")fs.PrintDefaults()fmt.Fprintf(os.Stderr, "\nBehavior:\n")fmt.Fprintf(os.Stderr, " - Filter is always 'BirdNET' (parsed from filename)\n")fmt.Fprintf(os.Stderr, " - If .data file exists with BirdNET filter: error (refuses to clobber)\n")fmt.Fprintf(os.Stderr, " - If .data file exists with different filter: merge segments\n")fmt.Fprintf(os.Stderr, " - Confidence (0.0-1.0) converted to certainty (0-100)\n")fmt.Fprintf(os.Stderr, "\nExamples:\n")fmt.Fprintf(os.Stderr, " skraak calls from-birda --folder ./recordings\n")fmt.Fprintf(os.Stderr, " skraak calls from-birda --file recording.BirdNET.results.csv\n")fmt.Fprintf(os.Stderr, " skraak calls from-birda --folder ./recordings --delete\n")}if err := fs.Parse(args); err != nil {os.Exit(1)}// Validate that either folder or file is specifiedif *folder == "" && *file == "" {fmt.Fprintf(os.Stderr, "Error: Either --folder or --file is required\n\n")fs.Usage()os.Exit(1)}input := tools.CallsFromBirdaInput{Folder: *folder,File: *file,Delete: *delete,ProgressHandler: func(processed, total int, message string) {if total > 0 {percent := float64(processed) / float64(total) * 100fmt.Fprintf(os.Stderr, "\rProcessing BirdNET files: %d/%d (%.0f%%)", processed, total, percent)if processed == total {fmt.Fprintf(os.Stderr, "\n")}}},}fmt.Fprintf(os.Stderr, "Importing BirdNET results\n")if *folder != "" {fmt.Fprintf(os.Stderr, "Folder: %s\n", *folder)} else {fmt.Fprintf(os.Stderr, "File: %s\n", *file)}if *delete {fmt.Fprintf(os.Stderr, "Delete source files: enabled\n")}output, err := tools.CallsFromBirda(input)if err != nil {fmt.Fprintf(os.Stderr, "Error: %v\n", err)os.Exit(1)}fmt.Fprintf(os.Stderr, "Processed %d BirdNET files\n", output.FilesProcessed)fmt.Fprintf(os.Stderr, "Found %d calls across %d species\n",output.TotalCalls, len(output.SpeciesCount))fmt.Fprintf(os.Stderr, "Data files written: %d, skipped: %d\n",output.DataFilesWritten, output.DataFilesSkipped)if *delete {fmt.Fprintf(os.Stderr, "Files deleted: %d\n", output.FilesDeleted)}// Output JSON to stdoutenc := json.NewEncoder(os.Stdout)enc.SetIndent("", " ")enc.Encode(output)}// runCallsFromRaven handles the "calls from-raven" subcommandfunc runCallsFromRaven(args []string) {fs := flag.NewFlagSet("calls from-raven", flag.ExitOnError)folder := fs.String("folder", "", "Folder containing Raven selection files")file := fs.String("file", "", "Single Raven selection file to process")delete := fs.Bool("delete", false, "Delete Raven files after processing")fs.Usage = func() {fmt.Fprintf(os.Stderr, "Usage: skraak calls from-raven [options]\n\n")fmt.Fprintf(os.Stderr, "Import Raven selections to .data files.\n")fmt.Fprintf(os.Stderr, "Reads *.selections.txt files and creates/merges .data files.\n\n")fmt.Fprintf(os.Stderr, "Options:\n")fs.PrintDefaults()fmt.Fprintf(os.Stderr, "\nBehavior:\n")fmt.Fprintf(os.Stderr, " - Filter is always 'Raven' (parsed from filename)\n")fmt.Fprintf(os.Stderr, " - If .data file exists with Raven filter: error (refuses to clobber)\n")fmt.Fprintf(os.Stderr, " - If .data file exists with different filter: merge segments\n")fmt.Fprintf(os.Stderr, " - Frequency range preserved from Raven selections\n")fmt.Fprintf(os.Stderr, " - Certainty defaults to 70 (no confidence metric in Raven)\n")fmt.Fprintf(os.Stderr, "\nExamples:\n")fmt.Fprintf(os.Stderr, " skraak calls from-raven --folder ./recordings\n")fmt.Fprintf(os.Stderr, " skraak calls from-raven --file recording.Table.1.selections.txt\n")fmt.Fprintf(os.Stderr, " skraak calls from-raven --folder ./recordings --delete\n")}if err := fs.Parse(args); err != nil {os.Exit(1)}// Validate that either folder or file is specifiedif *folder == "" && *file == "" {fmt.Fprintf(os.Stderr, "Error: Either --folder or --file is required\n\n")fs.Usage()os.Exit(1)}input := tools.CallsFromRavenInput{Folder: *folder,File: *file,Delete: *delete,ProgressHandler: func(processed, total int, message string) {if total > 0 {percent := float64(processed) / float64(total) * 100fmt.Fprintf(os.Stderr, "\rProcessing Raven files: %d/%d (%.0f%%)", processed, total, percent)if processed == total {fmt.Fprintf(os.Stderr, "\n")}}},}fmt.Fprintf(os.Stderr, "Importing Raven selections\n")if *folder != "" {fmt.Fprintf(os.Stderr, "Folder: %s\n", *folder)} else {fmt.Fprintf(os.Stderr, "File: %s\n", *file)}if *delete {fmt.Fprintf(os.Stderr, "Delete source files: enabled\n")}output, err := tools.CallsFromRaven(input)if err != nil {fmt.Fprintf(os.Stderr, "Error: %v\n", err)os.Exit(1)}fmt.Fprintf(os.Stderr, "Processed %d Raven files\n", output.FilesProcessed)fmt.Fprintf(os.Stderr, "Found %d calls across %d species\n",output.TotalCalls, len(output.SpeciesCount))fmt.Fprintf(os.Stderr, "Data files written: %d, skipped: %d\n",output.DataFilesWritten, output.DataFilesSkipped)if *delete {fmt.Fprintf(os.Stderr, "Files deleted: %d\n", output.FilesDeleted)}// Output JSON to stdoutenc := json.NewEncoder(os.Stdout)enc.SetIndent("", " ")enc.Encode(output)}
**Added:**- `tools/calls_from_birda.go` — BirdNET results file parser- `tools/calls_from_raven.go` — Raven selections file parser- `cmd/calls.go` — New subcommands `from-birda` and `from-raven`- `tools/calls_from_birda_raven_test.go` — 10 test cases**Commands:**```bash# BirdNET (filter always "BirdNET")./skraak calls from-birda --folder /path/to/recordings./skraak calls from-birda --file recording.BirdNET.results.csv [--delete]# Raven (filter always "Raven")./skraak calls from-raven --folder /path/to/recordings./skraak calls from-raven --file recording.Table.1.selections.txt [--delete]```**File formats:**- BirdNET: `*.BirdNET.results.csv` (CSV with BOM, columns: Start, End, Scientific name, Common name, Confidence, File)- Raven: `*.selections.txt` (Tab-separated, columns: Begin Time, End Time, Low Freq, High Freq, Species)**Behavior (same as from-preds):**- Filter is always parsed from filename (no `--filter` option)- No clobber: if filter already exists, error- Merge: if different filter exists, append segments- Confidence (BirdNET) converted from 0.0-1.0 to 0-100- Frequency range preserved from Raven selections- `--delete` option removes source files after successful import**Tests:** 10 new tests covering:- New .data file creation- Same filter rejection (no clobber)- Different filter merge- Delete option- Folder mode (BirdNET only)- Multiple selections (Raven only)