package tools
import (
	"bufio"
	"fmt"
	"os"
	"path/filepath"
	"sort"
	"strings"
	"skraak/utils"
)
// CallsFromRavenInput defines the input for the calls-from-raven tool
type CallsFromRavenInput struct {
	Folder          string          `json:"folder" jsonschema:"Folder containing Raven selection files"`
	File            string          `json:"file" jsonschema:"Single Raven selection file to process"`
	Delete          bool            `json:"delete" jsonschema:"Delete Raven files after processing"`
	ProgressHandler ProgressHandler `json:"-"` // Optional progress callback
}
// CallsFromRavenOutput defines the output for the calls-from-raven tool
type CallsFromRavenOutput struct {
	Calls            []ClusteredCall `json:"calls"`
	TotalCalls       int             `json:"total_calls"`
	SpeciesCount     map[string]int  `json:"species_count"`
	DataFilesWritten int             `json:"data_files_written"`
	DataFilesSkipped int             `json:"data_files_skipped"`
	FilesProcessed   int             `json:"files_processed"`
	FilesDeleted     int             `json:"files_deleted"`
	Filter           string          `json:"filter"`
	Error            *string         `json:"error,omitempty"`
}
// RavenSelection represents a single Raven selection
type RavenSelection struct {
	StartTime float64
	EndTime   float64
	FreqLow   float64
	FreqHigh  float64
	Species   string
}
// CallsFromRaven processes Raven selection files and writes .data files
func CallsFromRaven(input CallsFromRavenInput) (CallsFromRavenOutput, error) {
	var output CallsFromRavenOutput
	output.Filter = "Raven"
	// Collect Raven files to process
	var ravenFiles []string
	if input.File != "" {
		ravenFiles = []string{input.File}
	} else if input.Folder != "" {
		var err error
		ravenFiles, err = findRavenFiles(input.Folder)
		if err != nil {
			errMsg := fmt.Sprintf("Failed to find Raven files: %v", err)
			output.Error = &errMsg
			return output, fmt.Errorf("%s", errMsg)
		}
	} else {
		errMsg := "Either --folder or --file must be specified"
		output.Error = &errMsg
		return output, fmt.Errorf("%s", errMsg)
	}
	if len(ravenFiles) == 0 {
		errMsg := "No Raven files found"
		output.Error = &errMsg
		return output, fmt.Errorf("%s", errMsg)
	}
	// Process each Raven file
	speciesCount := make(map[string]int)
	var allCalls []ClusteredCall
	dataFilesWritten := 0
	dataFilesSkipped := 0
	filesProcessed := 0
	filesDeleted := 0
	for _, ravenFile := range ravenFiles {
		calls, written, skipped, err := processRavenFile(ravenFile)
		if err != nil {
			// Stop on first error
			errMsg := fmt.Sprintf("Error processing %s: %v", ravenFile, err)
			output.Error = &errMsg
			return output, fmt.Errorf("%s", errMsg)
		}
		if written {
			dataFilesWritten++
		}
		if skipped {
			dataFilesSkipped++
		}
		for _, call := range calls {
			allCalls = append(allCalls, call)
			speciesCount[call.EbirdCode]++
		}
		filesProcessed++
		// Delete if requested and successfully processed
		if input.Delete && written {
			if err := os.Remove(ravenFile); err != nil {
				errMsg := fmt.Sprintf("Failed to delete %s: %v", ravenFile, err)
				output.Error = &errMsg
				return output, fmt.Errorf("%s", errMsg)
			}
			filesDeleted++
		}
		if input.ProgressHandler != nil {
			input.ProgressHandler(filesProcessed, len(ravenFiles), filepath.Base(ravenFile))
		}
	}
	// Sort all calls by file, then start time
	sort.Slice(allCalls, func(i, j int) bool {
		if allCalls[i].File != allCalls[j].File {
			return allCalls[i].File < allCalls[j].File
		}
		return allCalls[i].StartTime < allCalls[j].StartTime
	})
	output.Calls = allCalls
	output.TotalCalls = len(allCalls)
	output.SpeciesCount = speciesCount
	output.DataFilesWritten = dataFilesWritten
	output.DataFilesSkipped = dataFilesSkipped
	output.FilesProcessed = filesProcessed
	output.FilesDeleted = filesDeleted
	return output, nil
}
// findRavenFiles finds all Raven selection files in a folder
func findRavenFiles(folder string) ([]string, error) {
	var files []string
	entries, err := os.ReadDir(folder)
	if err != nil {
		return nil, err
	}
	for _, entry := range entries {
		name := entry.Name()
		if strings.HasSuffix(name, ".selections.txt") {
			files = append(files, filepath.Join(folder, name))
		}
	}
	return files, nil
}
// processRavenFile processes a single Raven selection file
func processRavenFile(ravenFile string) ([]ClusteredCall, bool, bool, error) {
	// Open file
	file, err := os.Open(ravenFile)
	if err != nil {
		return nil, false, false, fmt.Errorf("failed to open file: %w", err)
	}
	defer file.Close()
	// Read header and selections (tab-separated)
	scanner := bufio.NewScanner(file)
	// Read header line
	if !scanner.Scan() {
		return nil, false, false, fmt.Errorf("empty file")
	}
	header := strings.Split(scanner.Text(), "\t")
	// Find column indices
	beginTimeIdx := -1
	endTimeIdx := -1
	lowFreqIdx := -1
	highFreqIdx := -1
	speciesIdx := -1
	for i, col := range header {
		switch col {
		case "Begin Time (s)":
			beginTimeIdx = i
		case "End Time (s)":
			endTimeIdx = i
		case "Low Freq (Hz)":
			lowFreqIdx = i
		case "High Freq (Hz)":
			highFreqIdx = i
		case "Species":
			speciesIdx = i
		}
	}
	if beginTimeIdx == -1 || endTimeIdx == -1 || speciesIdx == -1 {
		return nil, false, false, fmt.Errorf("missing required columns in Raven file")
	}
	// Read selections
	var selections []RavenSelection
	for scanner.Scan() {
		line := scanner.Text()
		if line == "" {
			continue
		}
		fields := strings.Split(line, "\t")
		if len(fields) <= speciesIdx {
			continue
		}
		var sel RavenSelection
		fmt.Sscanf(fields[beginTimeIdx], "%f", &sel.StartTime)
		fmt.Sscanf(fields[endTimeIdx], "%f", &sel.EndTime)
		if lowFreqIdx >= 0 && lowFreqIdx < len(fields) {
			fmt.Sscanf(fields[lowFreqIdx], "%f", &sel.FreqLow)
		}
		if highFreqIdx >= 0 && highFreqIdx < len(fields) {
			fmt.Sscanf(fields[highFreqIdx], "%f", &sel.FreqHigh)
		}
		sel.Species = fields[speciesIdx]
		selections = append(selections, sel)
	}
	if err := scanner.Err(); err != nil {
		return nil, false, false, fmt.Errorf("error reading file: %w", err)
	}
	if len(selections) == 0 {
		return nil, false, true, nil // No selections, skip
	}
	// Derive WAV path from Raven filename
	// "20230610_150000.Table.1.selections.txt" -> "20230610_150000.WAV"
	base := filepath.Base(ravenFile)
	// Remove .selections.txt
	nameWithoutSuffix := strings.TrimSuffix(base, ".selections.txt")
	// Remove .Table.X (or similar pattern)
	idx := strings.Index(nameWithoutSuffix, ".Table.")
	if idx > 0 {
		nameWithoutSuffix = nameWithoutSuffix[:idx]
	}
	wavBase := nameWithoutSuffix + ".WAV"
	wavPath := filepath.Join(filepath.Dir(ravenFile), wavBase)
	// Check if WAV exists (to get sample rate and duration)
	sampleRate, duration, err := utils.ParseWAVHeaderMinimal(wavPath)
	if err != nil {
		return nil, false, true, nil // Skip if WAV not found or invalid
	}
	dataPath := wavPath + ".data"
	// Convert selections to segments
	segments := buildRavenSegments(selections, sampleRate)
	// Build metadata
	meta := AviaNZMeta{
		Operator: "Raven",
		Duration: duration,
	}
	reviewer := "None"
	meta.Reviewer = &reviewer
	// Write .data file (safe write)
	if err := writeDotDataFileSafe(dataPath, segments, "Raven", meta); err != nil {
		return nil, false, false, err
	}
	// Convert to ClusteredCalls for output
	var calls []ClusteredCall
	for _, sel := range selections {
		calls = append(calls, ClusteredCall{
			File:      wavPath,
			StartTime: sel.StartTime,
			EndTime:   sel.EndTime,
			EbirdCode: sel.Species,
			Segments:  1,
		})
	}
	return calls, true, false, nil
}
// buildRavenSegments converts Raven selections to AviaNZ segments
func buildRavenSegments(selections []RavenSelection, sampleRate int) []AviaNZSegment {
	var segments []AviaNZSegment
	for _, sel := range selections {
		labels := []AviaNZLabel{
			{
				Species:   sel.Species,
				Certainty: 70, // Default certainty for Raven (no confidence metric)
				Filter:    "Raven",
			},
		}
		// Use frequency range from Raven, or full band if not specified
		freqLow := sel.FreqLow
		freqHigh := sel.FreqHigh
		if freqLow == 0 && freqHigh == 0 {
			freqHigh = float64(sampleRate)
		}
		segment := AviaNZSegment{
			sel.StartTime,
			sel.EndTime,
			freqLow,
			freqHigh,
			labels,
		}
		segments = append(segments, segment)
	}
	return segments
}

package tools
import (
	"os"
	"path/filepath"
	"testing"
	"skraak/utils"
)
// ============================================
// BirdNET Tests
// ============================================
func TestCallsFromBirda_NewDataFile(t *testing.T) {
	tmpDir := t.TempDir()
	// Create a minimal WAV file
	wavPath := filepath.Join(tmpDir, "test.WAV")
	createMinimalWAV(t, wavPath, 16000, 60.0)
	// Create BirdNET results file
	birdaPath := filepath.Join(tmpDir, "test.BirdNET.results.csv")
	birdaContent := "\ufeffStart (s),End (s),Scientific name,Common name,Confidence,File\n0.0,3.0,Turdus migratorius,American Robin,0.85,/some/path/test.WAV\n"
	if err := os.WriteFile(birdaPath, []byte(birdaContent), 0644); err != nil {
		t.Fatal(err)
	}
	input := CallsFromBirdaInput{
		File: birdaPath,
	}
	output, err := CallsFromBirda(input)
	if err != nil {
		t.Fatalf("unexpected error: %v", err)
	}
	if output.DataFilesWritten != 1 {
		t.Errorf("expected 1 data file written, got %d", output.DataFilesWritten)
	}
	if output.Filter != "BirdNET" {
		t.Errorf("expected filter 'BirdNET', got '%s'", output.Filter)
	}
	if output.TotalCalls != 1 {
		t.Errorf("expected 1 call, got %d", output.TotalCalls)
	}
	// Verify .data file was created
	dataPath := wavPath + ".data"
	df, err := utils.ParseDataFile(dataPath)
	if err != nil {
		t.Fatalf("failed to parse .data file: %v", err)
	}
	if len(df.Segments) != 1 {
		t.Errorf("expected 1 segment, got %d", len(df.Segments))
	}
	if df.Segments[0].Labels[0].Filter != "BirdNET" {
		t.Errorf("expected filter 'BirdNET', got '%s'", df.Segments[0].Labels[0].Filter)
	}
	if df.Segments[0].Labels[0].Certainty != 85 {
		t.Errorf("expected certainty 85, got %d", df.Segments[0].Labels[0].Certainty)
	}
}
func TestCallsFromBirda_ExistingSameFilter(t *testing.T) {
	tmpDir := t.TempDir()
	wavPath := filepath.Join(tmpDir, "test.WAV")
	createMinimalWAV(t, wavPath, 16000, 60.0)
	dataPath := wavPath + ".data"
	existingData := `[{"Operator": "Test", "Duration": 60.0}, [5.0, 10.0, 0, 16000, [{"species": "Existing Bird", "certainty": 90, "filter": "BirdNET"}]]]`
	if err := os.WriteFile(dataPath, []byte(existingData), 0644); err != nil {
		t.Fatal(err)
	}
	birdaPath := filepath.Join(tmpDir, "test.BirdNET.results.csv")
	birdaContent := "\ufeffStart (s),End (s),Scientific name,Common name,Confidence,File\n0.0,3.0,New Bird,New Bird,0.85,test.WAV\n"
	if err := os.WriteFile(birdaPath, []byte(birdaContent), 0644); err != nil {
		t.Fatal(err)
	}
	input := CallsFromBirdaInput{File: birdaPath}
	output, err := CallsFromBirda(input)
	if err == nil {
		t.Error("expected error for same filter, got nil")
	}
	if output.Error == nil {
		t.Error("expected error message in output")
	}
}
func TestCallsFromBirda_ExistingDifferentFilter(t *testing.T) {
	tmpDir := t.TempDir()
	wavPath := filepath.Join(tmpDir, "test.WAV")
	createMinimalWAV(t, wavPath, 16000, 60.0)
	dataPath := wavPath + ".data"
	existingData := `[{"Operator": "Test", "Duration": 60.0}, [5.0, 10.0, 0, 16000, [{"species": "Kiwi", "certainty": 90, "filter": "Manual"}]]]`
	if err := os.WriteFile(dataPath, []byte(existingData), 0644); err != nil {
		t.Fatal(err)
	}
	birdaPath := filepath.Join(tmpDir, "test.BirdNET.results.csv")
	birdaContent := "\ufeffStart (s),End (s),Scientific name,Common name,Confidence,File\n0.0,3.0,Robin,Robin,0.85,test.WAV\n"
	if err := os.WriteFile(birdaPath, []byte(birdaContent), 0644); err != nil {
		t.Fatal(err)
	}
	input := CallsFromBirdaInput{File: birdaPath}
	output, err := CallsFromBirda(input)
	if err != nil {
		t.Fatalf("unexpected error: %v", err)
	}
	if output.DataFilesWritten != 1 {
		t.Errorf("expected 1 data file written, got %d", output.DataFilesWritten)
	}
	df, err := utils.ParseDataFile(dataPath)
	if err != nil {
		t.Fatalf("failed to parse .data file: %v", err)
	}
	if len(df.Segments) != 2 {
		t.Errorf("expected 2 segments after merge, got %d", len(df.Segments))
	}
}
func TestCallsFromBirda_DeleteOption(t *testing.T) {
	tmpDir := t.TempDir()
	wavPath := filepath.Join(tmpDir, "test.WAV")
	createMinimalWAV(t, wavPath, 16000, 60.0)
	birdaPath := filepath.Join(tmpDir, "test.BirdNET.results.csv")
	birdaContent := "\ufeffStart (s),End (s),Scientific name,Common name,Confidence,File\n0.0,3.0,Robin,Robin,0.85,test.WAV\n"
	if err := os.WriteFile(birdaPath, []byte(birdaContent), 0644); err != nil {
		t.Fatal(err)
	}
	input := CallsFromBirdaInput{File: birdaPath, Delete: true}
	output, err := CallsFromBirda(input)
	if err != nil {
		t.Fatalf("unexpected error: %v", err)
	}
	if output.FilesDeleted != 1 {
		t.Errorf("expected 1 file deleted, got %d", output.FilesDeleted)
	}
	if _, err := os.Stat(birdaPath); !os.IsNotExist(err) {
		t.Error("expected BirdNET file to be deleted")
	}
}
func TestCallsFromBirda_FolderMode(t *testing.T) {
	tmpDir := t.TempDir()
	for i := 0; i < 2; i++ {
		wavPath := filepath.Join(tmpDir, "test"+string(rune('0'+i))+".WAV")
		createMinimalWAV(t, wavPath, 16000, 60.0)
		birdaPath := filepath.Join(tmpDir, "test"+string(rune('0'+i))+".BirdNET.results.csv")
		birdaContent := "\ufeffStart (s),End (s),Scientific name,Common name,Confidence,File\n0.0,3.0,Bird,Bird,0.85,test.WAV\n"
		if err := os.WriteFile(birdaPath, []byte(birdaContent), 0644); err != nil {
			t.Fatal(err)
		}
	}
	input := CallsFromBirdaInput{Folder: tmpDir}
	output, err := CallsFromBirda(input)
	if err != nil {
		t.Fatalf("unexpected error: %v", err)
	}
	if output.FilesProcessed != 2 {
		t.Errorf("expected 2 files processed, got %d", output.FilesProcessed)
	}
	if output.DataFilesWritten != 2 {
		t.Errorf("expected 2 data files written, got %d", output.DataFilesWritten)
	}
}
// ============================================
// Raven Tests
// ============================================
func TestCallsFromRaven_NewDataFile(t *testing.T) {
	tmpDir := t.TempDir()
	wavPath := filepath.Join(tmpDir, "test.WAV")
	createMinimalWAV(t, wavPath, 16000, 60.0)
	ravenPath := filepath.Join(tmpDir, "test.Table.1.selections.txt")
	ravenContent := "Selection\tView\tChannel\tBegin Time (s)\tEnd Time (s)\tLow Freq (Hz)\tHigh Freq (Hz)\tSpecies\n1\tSpectrogram 1\t1\t0.0\t5.0\t1000\t5000\tKiwi\n"
	if err := os.WriteFile(ravenPath, []byte(ravenContent), 0644); err != nil {
		t.Fatal(err)
	}
	input := CallsFromRavenInput{File: ravenPath}
	output, err := CallsFromRaven(input)
	if err != nil {
		t.Fatalf("unexpected error: %v", err)
	}
	if output.DataFilesWritten != 1 {
		t.Errorf("expected 1 data file written, got %d", output.DataFilesWritten)
	}
	if output.Filter != "Raven" {
		t.Errorf("expected filter 'Raven', got '%s'", output.Filter)
	}
	dataPath := wavPath + ".data"
	df, err := utils.ParseDataFile(dataPath)
	if err != nil {
		t.Fatalf("failed to parse .data file: %v", err)
	}
	if df.Segments[0].FreqLow != 1000 {
		t.Errorf("expected freq_low 1000, got %f", df.Segments[0].FreqLow)
	}
	if df.Segments[0].FreqHigh != 5000 {
		t.Errorf("expected freq_high 5000, got %f", df.Segments[0].FreqHigh)
	}
}
func TestCallsFromRaven_ExistingSameFilter(t *testing.T) {
	tmpDir := t.TempDir()
	wavPath := filepath.Join(tmpDir, "test.WAV")
	createMinimalWAV(t, wavPath, 16000, 60.0)
	dataPath := wavPath + ".data"
	existingData := `[{"Operator": "Test", "Duration": 60.0}, [5.0, 10.0, 0, 16000, [{"species": "Existing", "certainty": 90, "filter": "Raven"}]]]`
	if err := os.WriteFile(dataPath, []byte(existingData), 0644); err != nil {
		t.Fatal(err)
	}
	ravenPath := filepath.Join(tmpDir, "test.Table.1.selections.txt")
	ravenContent := "Selection\tView\tChannel\tBegin Time (s)\tEnd Time (s)\tLow Freq (Hz)\tHigh Freq (Hz)\tSpecies\n1\tSpectrogram 1\t1\t0.0\t5.0\t1000\t5000\tNew\n"
	if err := os.WriteFile(ravenPath, []byte(ravenContent), 0644); err != nil {
		t.Fatal(err)
	}
	input := CallsFromRavenInput{File: ravenPath}
	output, err := CallsFromRaven(input)
	if err == nil {
		t.Error("expected error for same filter, got nil")
	}
	if output.Error == nil {
		t.Error("expected error message in output")
	}
}
func TestCallsFromRaven_ExistingDifferentFilter(t *testing.T) {
	tmpDir := t.TempDir()
	wavPath := filepath.Join(tmpDir, "test.WAV")
	createMinimalWAV(t, wavPath, 16000, 60.0)
	dataPath := wavPath + ".data"
	existingData := `[{"Operator": "Test", "Duration": 60.0}, [5.0, 10.0, 0, 16000, [{"species": "Kiwi", "certainty": 90, "filter": "BirdNET"}]]]`
	if err := os.WriteFile(dataPath, []byte(existingData), 0644); err != nil {
		t.Fatal(err)
	}
	ravenPath := filepath.Join(tmpDir, "test.Table.1.selections.txt")
	ravenContent := "Selection\tView\tChannel\tBegin Time (s)\tEnd Time (s)\tLow Freq (Hz)\tHigh Freq (Hz)\tSpecies\n1\tSpectrogram 1\t1\t0.0\t5.0\t1000\t5000\tMorepork\n"
	if err := os.WriteFile(ravenPath, []byte(ravenContent), 0644); err != nil {
		t.Fatal(err)
	}
	input := CallsFromRavenInput{File: ravenPath}
	output, err := CallsFromRaven(input)
	if err != nil {
		t.Fatalf("unexpected error: %v", err)
	}
	if output.DataFilesWritten != 1 {
		t.Errorf("expected 1 data file written, got %d", output.DataFilesWritten)
	}
	df, err := utils.ParseDataFile(dataPath)
	if err != nil {
		t.Fatalf("failed to parse .data file: %v", err)
	}
	if len(df.Segments) != 2 {
		t.Errorf("expected 2 segments after merge, got %d", len(df.Segments))
	}
}
func TestCallsFromRaven_DeleteOption(t *testing.T) {
	tmpDir := t.TempDir()
	wavPath := filepath.Join(tmpDir, "test.WAV")
	createMinimalWAV(t, wavPath, 16000, 60.0)
	ravenPath := filepath.Join(tmpDir, "test.Table.1.selections.txt")
	ravenContent := "Selection\tView\tChannel\tBegin Time (s)\tEnd Time (s)\tLow Freq (Hz)\tHigh Freq (Hz)\tSpecies\n1\tSpectrogram 1\t1\t0.0\t5.0\t1000\t5000\tKiwi\n"
	if err := os.WriteFile(ravenPath, []byte(ravenContent), 0644); err != nil {
		t.Fatal(err)
	}
	input := CallsFromRavenInput{File: ravenPath, Delete: true}
	output, err := CallsFromRaven(input)
	if err != nil {
		t.Fatalf("unexpected error: %v", err)
	}
	if output.FilesDeleted != 1 {
		t.Errorf("expected 1 file deleted, got %d", output.FilesDeleted)
	}
	if _, err := os.Stat(ravenPath); !os.IsNotExist(err) {
		t.Error("expected Raven file to be deleted")
	}
}
func TestCallsFromRaven_MultipleSelections(t *testing.T) {
	tmpDir := t.TempDir()
	wavPath := filepath.Join(tmpDir, "test.WAV")
	createMinimalWAV(t, wavPath, 16000, 60.0)
	ravenPath := filepath.Join(tmpDir, "test.Table.1.selections.txt")
	ravenContent := "Selection\tView\tChannel\tBegin Time (s)\tEnd Time (s)\tLow Freq (Hz)\tHigh Freq (Hz)\tSpecies\n1\tSpectrogram 1\t1\t0.0\t5.0\t1000\t5000\tKiwi\n2\tSpectrogram 1\t1\t10.0\t15.0\t2000\t6000\tMorepork\n3\tSpectrogram 1\t1\t20.0\t25.0\t1500\t4500\tTui\n"
	if err := os.WriteFile(ravenPath, []byte(ravenContent), 0644); err != nil {
		t.Fatal(err)
	}
	input := CallsFromRavenInput{File: ravenPath}
	output, err := CallsFromRaven(input)
	if err != nil {
		t.Fatalf("unexpected error: %v", err)
	}
	if output.TotalCalls != 3 {
		t.Errorf("expected 3 calls, got %d", output.TotalCalls)
	}
	if output.SpeciesCount["Kiwi"] != 1 || output.SpeciesCount["Morepork"] != 1 || output.SpeciesCount["Tui"] != 1 {
		t.Errorf("unexpected species count: %v", output.SpeciesCount)
	}
}

package tools
import (
	"encoding/csv"
	"fmt"
	"io"
	"os"
	"path/filepath"
	"sort"
	"strings"
	"skraak/utils"
)
// CallsFromBirdaInput defines the input for the calls-from-birda tool
type CallsFromBirdaInput struct {
	Folder          string          `json:"folder" jsonschema:"Folder containing BirdNET results files"`
	File            string          `json:"file" jsonschema:"Single BirdNET results file to process"`
	Delete          bool            `json:"delete" jsonschema:"Delete BirdNET files after processing"`
	ProgressHandler ProgressHandler `json:"-"` // Optional progress callback
}
// CallsFromBirdaOutput defines the output for the calls-from-birda tool
type CallsFromBirdaOutput struct {
	Calls            []ClusteredCall `json:"calls"`
	TotalCalls       int             `json:"total_calls"`
	SpeciesCount     map[string]int  `json:"species_count"`
	DataFilesWritten int             `json:"data_files_written"`
	DataFilesSkipped int             `json:"data_files_skipped"`
	FilesProcessed   int             `json:"files_processed"`
	FilesDeleted     int             `json:"files_deleted"`
	Filter           string          `json:"filter"`
	Error            *string         `json:"error,omitempty"`
}
// BirdNETDetection represents a single BirdNET detection
type BirdNETDetection struct {
	StartTime     float64
	EndTime       float64
	ScientificName string
	CommonName    string
	Confidence    float64
	WAVPath       string
}
// CallsFromBirda processes BirdNET results files and writes .data files
func CallsFromBirda(input CallsFromBirdaInput) (CallsFromBirdaOutput, error) {
	var output CallsFromBirdaOutput
	output.Filter = "BirdNET"
	// Collect BirdNET files to process
	var birdaFiles []string
	if input.File != "" {
		birdaFiles = []string{input.File}
	} else if input.Folder != "" {
		var err error
		birdaFiles, err = findBirdaFiles(input.Folder)
		if err != nil {
			errMsg := fmt.Sprintf("Failed to find BirdNET files: %v", err)
			output.Error = &errMsg
			return output, fmt.Errorf("%s", errMsg)
		}
	} else {
		errMsg := "Either --folder or --file must be specified"
		output.Error = &errMsg
		return output, fmt.Errorf("%s", errMsg)
	}
	if len(birdaFiles) == 0 {
		errMsg := "No BirdNET files found"
		output.Error = &errMsg
		return output, fmt.Errorf("%s", errMsg)
	}
	// Process each BirdNET file
	speciesCount := make(map[string]int)
	var allCalls []ClusteredCall
	dataFilesWritten := 0
	dataFilesSkipped := 0
	filesProcessed := 0
	filesDeleted := 0
	for _, birdaFile := range birdaFiles {
		calls, written, skipped, err := processBirdaFile(birdaFile)
		if err != nil {
			// Stop on first error
			errMsg := fmt.Sprintf("Error processing %s: %v", birdaFile, err)
			output.Error = &errMsg
			return output, fmt.Errorf("%s", errMsg)
		}
		if written {
			dataFilesWritten++
		}
		if skipped {
			dataFilesSkipped++
		}
		for _, call := range calls {
			allCalls = append(allCalls, call)
			speciesCount[call.EbirdCode]++
		}
		filesProcessed++
		// Delete if requested and successfully processed
		if input.Delete && written {
			if err := os.Remove(birdaFile); err != nil {
				errMsg := fmt.Sprintf("Failed to delete %s: %v", birdaFile, err)
				output.Error = &errMsg
				return output, fmt.Errorf("%s", errMsg)
			}
			filesDeleted++
		}
		if input.ProgressHandler != nil {
			input.ProgressHandler(filesProcessed, len(birdaFiles), filepath.Base(birdaFile))
		}
	}
	// Sort all calls by file, then start time
	sort.Slice(allCalls, func(i, j int) bool {
		if allCalls[i].File != allCalls[j].File {
			return allCalls[i].File < allCalls[j].File
		}
		return allCalls[i].StartTime < allCalls[j].StartTime
	})
	output.Calls = allCalls
	output.TotalCalls = len(allCalls)
	output.SpeciesCount = speciesCount
	output.DataFilesWritten = dataFilesWritten
	output.DataFilesSkipped = dataFilesSkipped
	output.FilesProcessed = filesProcessed
	output.FilesDeleted = filesDeleted
	return output, nil
}
// findBirdaFiles finds all BirdNET results files in a folder
func findBirdaFiles(folder string) ([]string, error) {
	var files []string
	entries, err := os.ReadDir(folder)
	if err != nil {
		return nil, err
	}
	for _, entry := range entries {
		name := entry.Name()
		if strings.HasSuffix(name, ".BirdNET.results.csv") {
			files = append(files, filepath.Join(folder, name))
		}
	}
	return files, nil
}
// processBirdaFile processes a single BirdNET results file
func processBirdaFile(birdaFile string) ([]ClusteredCall, bool, bool, error) {
	// Open and parse CSV
	file, err := os.Open(birdaFile)
	if err != nil {
		return nil, false, false, fmt.Errorf("failed to open file: %w", err)
	}
	defer file.Close()
	// Create CSV reader
	reader := csv.NewReader(file)
	// Read header
	header, err := reader.Read()
	if err != nil {
		return nil, false, false, fmt.Errorf("failed to read header: %w", err)
	}
	// Find column indices (handle BOM prefix)
	startIdx := -1
	endIdx := -1
	commonNameIdx := -1
	confidenceIdx := -1
	fileIdx := -1
	for i, col := range header {
		// Remove BOM if present
		col = strings.TrimPrefix(col, "\ufeff")
		switch col {
		case "Start (s)":
			startIdx = i
		case "End (s)":
			endIdx = i
		case "Common name":
			commonNameIdx = i
		case "Confidence":
			confidenceIdx = i
		case "File":
			fileIdx = i
		}
	}
	if startIdx == -1 || endIdx == -1 || commonNameIdx == -1 || confidenceIdx == -1 {
		return nil, false, false, fmt.Errorf("missing required columns in BirdNET file")
	}
	// Read detections
	var detections []BirdNETDetection
	for {
		record, err := reader.Read()
		if err == io.EOF {
			break
		}
		if err != nil {
			return nil, false, false, fmt.Errorf("failed to read record: %w", err)
		}
		var det BirdNETDetection
		fmt.Sscanf(record[startIdx], "%f", &det.StartTime)
		fmt.Sscanf(record[endIdx], "%f", &det.EndTime)
		det.CommonName = record[commonNameIdx]
		fmt.Sscanf(record[confidenceIdx], "%f", &det.Confidence)
		if fileIdx >= 0 && fileIdx < len(record) {
			det.WAVPath = record[fileIdx]
		}
		detections = append(detections, det)
	}
	if len(detections) == 0 {
		return nil, false, true, nil // No detections, skip
	}
	// Determine WAV path and .data path
	var wavPath string
	if detections[0].WAVPath != "" {
		// Check if the path from File column exists
		if _, err := os.Stat(detections[0].WAVPath); err == nil {
			wavPath = detections[0].WAVPath
		} else {
			// Derive from BirdNET filename (File column may have wrong path)
			base := filepath.Base(birdaFile)
			wavBase := strings.TrimSuffix(base, ".BirdNET.results.csv") + ".WAV"
			wavPath = filepath.Join(filepath.Dir(birdaFile), wavBase)
		}
	} else {
		// Derive from BirdNET filename
		base := filepath.Base(birdaFile)
		wavBase := strings.TrimSuffix(base, ".BirdNET.results.csv") + ".WAV"
		wavPath = filepath.Join(filepath.Dir(birdaFile), wavBase)
	}
	// Check if WAV exists (to get sample rate and duration)
	sampleRate, duration, err := utils.ParseWAVHeaderMinimal(wavPath)
	if err != nil {
		return nil, false, true, nil // Skip if WAV not found or invalid
	}
	dataPath := wavPath + ".data"
	// Convert detections to segments
	segments := buildBirdNETSegments(detections, sampleRate)
	// Build metadata
	meta := AviaNZMeta{
		Operator: "BirdNET",
		Duration: duration,
	}
	reviewer := "None"
	meta.Reviewer = &reviewer
	// Write .data file (safe write)
	if err := writeDotDataFileSafe(dataPath, segments, "BirdNET", meta); err != nil {
		return nil, false, false, err
	}
	// Convert to ClusteredCalls for output
	var calls []ClusteredCall
	for _, det := range detections {
		calls = append(calls, ClusteredCall{
			File:      wavPath,
			StartTime: det.StartTime,
			EndTime:   det.EndTime,
			EbirdCode: det.CommonName,
			Segments:  1,
		})
	}
	return calls, true, false, nil
}
// buildBirdNETSegments converts BirdNET detections to AviaNZ segments
func buildBirdNETSegments(detections []BirdNETDetection, sampleRate int) []AviaNZSegment {
	var segments []AviaNZSegment
	for _, det := range detections {
		// Convert confidence (0.0-1.0) to certainty (0-100)
		certainty := int(det.Confidence * 100)
		if certainty < 0 {
			certainty = 0
		}
		if certainty > 100 {
			certainty = 100
		}
		labels := []AviaNZLabel{
			{
				Species:   det.CommonName,
				Certainty: certainty,
				Filter:    "BirdNET",
			},
		}
		segment := AviaNZSegment{
			det.StartTime,
			det.EndTime,
			0,          // freq_low
			sampleRate, // freq_high (full band)
			labels,
		}
		segments = append(segments, segment)
	}
	return segments
}

PLAN MODE: Great, now that we have done that, lets implemehnt 2 new calls commands:
calls from-birda --folder (or just 1 file with --file) --delete (optional, to delete the birda files as we go)
calls from-raven --folder (or just 1 file with --file) --delete (optional, to delete the raven file as we go)
We need to append birda and raven files to existing .data files, or create a new .data if not present. 
find sample birda files (1 or 0, per WAV) at /media/david/SSD4/Twenty_Four_Seven/R620/2024-05-06/20230610_150000.BirdNET.results.csv (note .BirdNET.results.csv) FILTER=BirdNET
find sample Raven files (1 or 0, per WAV) at /media/david/SSD4/Twenty_Four_Seven/R620/2024-05-06/20230610_150000.Table.1.selections.txt (note Table.$x.selections.txt, could be 1, 2, 3 ..., usually just 1, look for *.selections.txt) FILTER=Raven
Note that both types of csv files are similar to .data files, they refer to a WAV file.
Behaviour must be the same as calls from-preds, no clobber, we do not allow birda or raven files to be imported more than 1 time
We do not allow manual overide for --filter, we always parse from the filename.
filter BirdNET from *.BirdNET.results.csv
filter Raven from *.selections.txt
both new commands should output a json summary of calls parsed in the sae form as the output from calls from-preds cmd
PLAN MODE

	case "from-birda":
		runCallsFromBirda(args[1:])
	case "from-raven":
		runCallsFromRaven(args[1:])

	fmt.Fprintf(os.Stderr, "  from-birda   Import BirdNET results to .data files\n")
	fmt.Fprintf(os.Stderr, "  from-raven   Import Raven selections to .data files\n")

	fmt.Fprintf(os.Stderr, "  skraak calls from-preds --csv preds.csv --dot-data=false > calls.json\n")
	fmt.Fprintf(os.Stderr, "  skraak calls from-preds --csv preds.csv --filter my-filter\n")

	fmt.Fprintf(os.Stderr, "  skraak calls from-birda --folder ./recordings\n")
	fmt.Fprintf(os.Stderr, "  skraak calls from-raven --folder ./recordings --delete\n")

	fmt.Fprintf(os.Stderr, "  skraak calls classify --folder ./data --reviewer David \\\n")
	fmt.Fprintf(os.Stderr, "      --bind k=Kiwi --bind d='Kiwi+Duet' --bind n='Don''t Know'\n")

// runCallsFromBirda handles the "calls from-birda" subcommand
func runCallsFromBirda(args []string) {
	fs := flag.NewFlagSet("calls from-birda", flag.ExitOnError)
	folder := fs.String("folder", "", "Folder containing BirdNET results files")
	file := fs.String("file", "", "Single BirdNET results file to process")
	delete := fs.Bool("delete", false, "Delete BirdNET files after processing")
	fs.Usage = func() {
		fmt.Fprintf(os.Stderr, "Usage: skraak calls from-birda [options]\n\n")
		fmt.Fprintf(os.Stderr, "Import BirdNET results to .data files.\n")
		fmt.Fprintf(os.Stderr, "Reads *.BirdNET.results.csv files and creates/merges .data files.\n\n")
		fmt.Fprintf(os.Stderr, "Options:\n")
		fs.PrintDefaults()
		fmt.Fprintf(os.Stderr, "\nBehavior:\n")
		fmt.Fprintf(os.Stderr, "  - Filter is always 'BirdNET' (parsed from filename)\n")
		fmt.Fprintf(os.Stderr, "  - If .data file exists with BirdNET filter: error (refuses to clobber)\n")
		fmt.Fprintf(os.Stderr, "  - If .data file exists with different filter: merge segments\n")
		fmt.Fprintf(os.Stderr, "  - Confidence (0.0-1.0) converted to certainty (0-100)\n")
		fmt.Fprintf(os.Stderr, "\nExamples:\n")
		fmt.Fprintf(os.Stderr, "  skraak calls from-birda --folder ./recordings\n")
		fmt.Fprintf(os.Stderr, "  skraak calls from-birda --file recording.BirdNET.results.csv\n")
		fmt.Fprintf(os.Stderr, "  skraak calls from-birda --folder ./recordings --delete\n")
	}
	if err := fs.Parse(args); err != nil {
		os.Exit(1)
	}
	// Validate that either folder or file is specified
	if *folder == "" && *file == "" {
		fmt.Fprintf(os.Stderr, "Error: Either --folder or --file is required\n\n")
		fs.Usage()
		os.Exit(1)
	}
	input := tools.CallsFromBirdaInput{
		Folder: *folder,
		File:   *file,
		Delete: *delete,
		ProgressHandler: func(processed, total int, message string) {
			if total > 0 {
				percent := float64(processed) / float64(total) * 100
				fmt.Fprintf(os.Stderr, "\rProcessing BirdNET files: %d/%d (%.0f%%)", processed, total, percent)
				if processed == total {
					fmt.Fprintf(os.Stderr, "\n")
				}
			}
		},
	}
	fmt.Fprintf(os.Stderr, "Importing BirdNET results\n")
	if *folder != "" {
		fmt.Fprintf(os.Stderr, "Folder: %s\n", *folder)
	} else {
		fmt.Fprintf(os.Stderr, "File: %s\n", *file)
	}
	if *delete {
		fmt.Fprintf(os.Stderr, "Delete source files: enabled\n")
	}
	output, err := tools.CallsFromBirda(input)
	if err != nil {
		fmt.Fprintf(os.Stderr, "Error: %v\n", err)
		os.Exit(1)
	}
	fmt.Fprintf(os.Stderr, "Processed %d BirdNET files\n", output.FilesProcessed)
	fmt.Fprintf(os.Stderr, "Found %d calls across %d species\n",
		output.TotalCalls, len(output.SpeciesCount))
	fmt.Fprintf(os.Stderr, "Data files written: %d, skipped: %d\n",
		output.DataFilesWritten, output.DataFilesSkipped)
	if *delete {
		fmt.Fprintf(os.Stderr, "Files deleted: %d\n", output.FilesDeleted)
	}
	// Output JSON to stdout
	enc := json.NewEncoder(os.Stdout)
	enc.SetIndent("", "  ")
	enc.Encode(output)
}
// runCallsFromRaven handles the "calls from-raven" subcommand
func runCallsFromRaven(args []string) {
	fs := flag.NewFlagSet("calls from-raven", flag.ExitOnError)
	folder := fs.String("folder", "", "Folder containing Raven selection files")
	file := fs.String("file", "", "Single Raven selection file to process")
	delete := fs.Bool("delete", false, "Delete Raven files after processing")
	fs.Usage = func() {
		fmt.Fprintf(os.Stderr, "Usage: skraak calls from-raven [options]\n\n")
		fmt.Fprintf(os.Stderr, "Import Raven selections to .data files.\n")
		fmt.Fprintf(os.Stderr, "Reads *.selections.txt files and creates/merges .data files.\n\n")
		fmt.Fprintf(os.Stderr, "Options:\n")
		fs.PrintDefaults()
		fmt.Fprintf(os.Stderr, "\nBehavior:\n")
		fmt.Fprintf(os.Stderr, "  - Filter is always 'Raven' (parsed from filename)\n")
		fmt.Fprintf(os.Stderr, "  - If .data file exists with Raven filter: error (refuses to clobber)\n")
		fmt.Fprintf(os.Stderr, "  - If .data file exists with different filter: merge segments\n")
		fmt.Fprintf(os.Stderr, "  - Frequency range preserved from Raven selections\n")
		fmt.Fprintf(os.Stderr, "  - Certainty defaults to 70 (no confidence metric in Raven)\n")
		fmt.Fprintf(os.Stderr, "\nExamples:\n")
		fmt.Fprintf(os.Stderr, "  skraak calls from-raven --folder ./recordings\n")
		fmt.Fprintf(os.Stderr, "  skraak calls from-raven --file recording.Table.1.selections.txt\n")
		fmt.Fprintf(os.Stderr, "  skraak calls from-raven --folder ./recordings --delete\n")
	}
	if err := fs.Parse(args); err != nil {
		os.Exit(1)
	}
	// Validate that either folder or file is specified
	if *folder == "" && *file == "" {
		fmt.Fprintf(os.Stderr, "Error: Either --folder or --file is required\n\n")
		fs.Usage()
		os.Exit(1)
	}
	input := tools.CallsFromRavenInput{
		Folder: *folder,
		File:   *file,
		Delete: *delete,
		ProgressHandler: func(processed, total int, message string) {
			if total > 0 {
				percent := float64(processed) / float64(total) * 100
				fmt.Fprintf(os.Stderr, "\rProcessing Raven files: %d/%d (%.0f%%)", processed, total, percent)
				if processed == total {
					fmt.Fprintf(os.Stderr, "\n")
				}
			}
		},
	}
	fmt.Fprintf(os.Stderr, "Importing Raven selections\n")
	if *folder != "" {
		fmt.Fprintf(os.Stderr, "Folder: %s\n", *folder)
	} else {
		fmt.Fprintf(os.Stderr, "File: %s\n", *file)
	}
	if *delete {
		fmt.Fprintf(os.Stderr, "Delete source files: enabled\n")
	}
	output, err := tools.CallsFromRaven(input)
	if err != nil {
		fmt.Fprintf(os.Stderr, "Error: %v\n", err)
		os.Exit(1)
	}
	fmt.Fprintf(os.Stderr, "Processed %d Raven files\n", output.FilesProcessed)
	fmt.Fprintf(os.Stderr, "Found %d calls across %d species\n",
		output.TotalCalls, len(output.SpeciesCount))
	fmt.Fprintf(os.Stderr, "Data files written: %d, skipped: %d\n",
		output.DataFilesWritten, output.DataFilesSkipped)
	if *delete {
		fmt.Fprintf(os.Stderr, "Files deleted: %d\n", output.FilesDeleted)
	}
	// Output JSON to stdout
	enc := json.NewEncoder(os.Stdout)
	enc.SetIndent("", "  ")
	enc.Encode(output)
}

## [2026-03-09] New Commands: calls from-birda and calls from-raven
**New feature:** Import BirdNET and Raven annotation files to .data files.

**Added:**
- `tools/calls_from_birda.go` — BirdNET results file parser
- `tools/calls_from_raven.go` — Raven selections file parser
- `cmd/calls.go` — New subcommands `from-birda` and `from-raven`
- `tools/calls_from_birda_raven_test.go` — 10 test cases
**Commands:**
```bash
# BirdNET (filter always "BirdNET")
./skraak calls from-birda --folder /path/to/recordings
./skraak calls from-birda --file recording.BirdNET.results.csv [--delete]
# Raven (filter always "Raven")
./skraak calls from-raven --folder /path/to/recordings
./skraak calls from-raven --file recording.Table.1.selections.txt [--delete]
```
**File formats:**
- BirdNET: `*.BirdNET.results.csv` (CSV with BOM, columns: Start, End, Scientific name, Common name, Confidence, File)
- Raven: `*.selections.txt` (Tab-separated, columns: Begin Time, End Time, Low Freq, High Freq, Species)
**Behavior (same as from-preds):**
- Filter is always parsed from filename (no `--filter` option)
- No clobber: if filter already exists, error
- Merge: if different filter exists, append segments
- Confidence (BirdNET) converted from 0.0-1.0 to 0-100
- Frequency range preserved from Raven selections
- `--delete` option removes source files after successful import
**Tests:** 10 new tests covering:
- New .data file creation
- Same filter rejection (no clobber)
- Different filter merge
- Delete option
- Folder mode (BirdNET only)
- Multiple selections (Raven only)