KLUEQ6X5CXVBV3KLJKEHWQYHIU6AYPP2WT4PWKM2QZJ7SNACCJ6QC t.Run("should parse a valid structured AudioMoth comment", func(t *testing.T) {comment := "Recorded at 21:00:00 24/02/2025 (UTC+13) by AudioMoth 248AB50153AB0549 at medium gain while battery was 4.3V and temperature was 15.8C."
t.Run("should parse a valid structured AudioMoth comment", testParseStructuredComment)t.Run("should return error for invalid comments", testParseInvalidComments)t.Run("should handle different timezone formats", testParseTimezoneFormats)t.Run("should parse all gain levels", testParseAllGainLevels)t.Run("should handle negative temperatures", testParseNegativeTemp)t.Run("should fallback to legacy parsing", testParseLegacyFallback)}
result, err := ParseAudioMothComment(comment)if err != nil {t.Fatalf("Failed to parse comment: %v", err)}
func testParseStructuredComment(t *testing.T) {comment := "Recorded at 21:00:00 24/02/2025 (UTC+13) by AudioMoth 248AB50153AB0549 at medium gain while battery was 4.3V and temperature was 15.8C."
// Check timestamp (should be in UTC+13)expected := time.Date(2025, 2, 24, 21, 0, 0, 0, time.FixedZone("UTC+13", 13*3600))if !result.Timestamp.Equal(expected) {t.Errorf("Timestamp incorrect: got %v, want %v", result.Timestamp, expected)}
result, err := ParseAudioMothComment(comment)if err != nil {t.Fatalf("Failed to parse comment: %v", err)}
// Convert to UTC and verifyutc := result.Timestamp.UTC()expectedUTC := time.Date(2025, 2, 24, 8, 0, 0, 0, time.UTC)if !utc.Equal(expectedUTC) {t.Errorf("UTC timestamp incorrect: got %v, want %v", utc, expectedUTC)}
expected := time.Date(2025, 2, 24, 21, 0, 0, 0, time.FixedZone("UTC+13", 13*3600))if !result.Timestamp.Equal(expected) {t.Errorf("Timestamp incorrect: got %v, want %v", result.Timestamp, expected)}
if result.RecorderID != "248AB50153AB0549" {t.Errorf("RecorderID incorrect: got %s, want 248AB50153AB0549", result.RecorderID)}
utc := result.Timestamp.UTC()expectedUTC := time.Date(2025, 2, 24, 8, 0, 0, 0, time.UTC)if !utc.Equal(expectedUTC) {t.Errorf("UTC timestamp incorrect: got %v, want %v", utc, expectedUTC)}
if result.Gain != db.GainMedium {t.Errorf("Gain incorrect: got %s, want %s", result.Gain, db.GainMedium)}
if result.RecorderID != "248AB50153AB0549" {t.Errorf("RecorderID incorrect: got %s, want 248AB50153AB0549", result.RecorderID)}if result.Gain != db.GainMedium {t.Errorf("Gain incorrect: got %s, want %s", result.Gain, db.GainMedium)}if result.BatteryV != 4.3 {t.Errorf("BatteryV incorrect: got %f, want 4.3", result.BatteryV)}if result.TempC != 15.8 {t.Errorf("TempC incorrect: got %f, want 15.8", result.TempC)}}
if result.BatteryV != 4.3 {t.Errorf("BatteryV incorrect: got %f, want 4.3", result.BatteryV)}
func testParseInvalidComments(t *testing.T) {invalidComments := []string{"Not an AudioMoth comment","Recorded at invalid time format","Short comment","","AudioMoth without proper format",}
if result.TempC != 15.8 {t.Errorf("TempC incorrect: got %f, want 15.8", result.TempC)
for _, comment := range invalidComments {_, err := ParseAudioMothComment(comment)if err == nil {t.Errorf("Expected error for invalid comment: %s", comment)
t.Run("should return error for invalid comments", func(t *testing.T) {invalidComments := []string{"Not an AudioMoth comment","Recorded at invalid time format","Short comment","","AudioMoth without proper format",}
func testParseTimezoneFormats(t *testing.T) {commentUTCMinus := "Recorded at 10:30:45 15/06/2024 (UTC-5) by AudioMoth 123456789ABCDEF0 at high gain while battery was 3.9V and temperature was 22.1C."
for _, comment := range invalidComments {_, err := ParseAudioMothComment(comment)if err == nil {t.Errorf("Expected error for invalid comment: %s", comment)}}})
result, err := ParseAudioMothComment(commentUTCMinus)if err != nil {t.Fatalf("Failed to parse comment: %v", err)}
t.Run("should handle different timezone formats", func(t *testing.T) {commentUTCMinus := "Recorded at 10:30:45 15/06/2024 (UTC-5) by AudioMoth 123456789ABCDEF0 at high gain while battery was 3.9V and temperature was 22.1C."result, err := ParseAudioMothComment(commentUTCMinus)if err != nil {t.Fatalf("Failed to parse comment: %v", err)}// Check timestamp is in UTC-5expected := time.Date(2024, 6, 15, 10, 30, 45, 0, time.FixedZone("UTC-5", -5*3600))if !result.Timestamp.Equal(expected) {t.Errorf("Timestamp incorrect: got %v, want %v", result.Timestamp, expected)}if result.Gain != db.GainHigh {t.Errorf("Gain incorrect: got %s, want %s", result.Gain, db.GainHigh)}if result.BatteryV != 3.9 {t.Errorf("BatteryV incorrect: got %f, want 3.9", result.BatteryV)}
expected := time.Date(2024, 6, 15, 10, 30, 45, 0, time.FixedZone("UTC-5", -5*3600))if !result.Timestamp.Equal(expected) {t.Errorf("Timestamp incorrect: got %v, want %v", result.Timestamp, expected)}if result.Gain != db.GainHigh {t.Errorf("Gain incorrect: got %s, want %s", result.Gain, db.GainHigh)}if result.BatteryV != 3.9 {t.Errorf("BatteryV incorrect: got %f, want 3.9", result.BatteryV)}if result.TempC != 22.1 {t.Errorf("TempC incorrect: got %f, want 22.1", result.TempC)}}
if result.TempC != 22.1 {t.Errorf("TempC incorrect: got %f, want 22.1", result.TempC)}})t.Run("should parse all gain levels", func(t *testing.T) {testCases := []struct {gainStr stringexpected db.GainLevel}{{"low", db.GainLow},{"low-medium", db.GainLowMedium},{"medium", db.GainMedium},{"medium-high", db.GainMediumHigh},{"high", db.GainHigh},}
func testParseAllGainLevels(t *testing.T) {testCases := []struct {gainStr stringexpected db.GainLevel}{{"low", db.GainLow},{"low-medium", db.GainLowMedium},{"medium", db.GainMedium},{"medium-high", db.GainMediumHigh},{"high", db.GainHigh},}
for _, tc := range testCases {comment := "Recorded at 21:00:00 24/02/2025 (UTC+13) by AudioMoth 248AB50153AB0549 at " + tc.gainStr + " gain while battery was 4.3V and temperature was 15.8C."result, err := ParseAudioMothComment(comment)if err != nil {t.Errorf("Failed to parse comment with gain %s: %v", tc.gainStr, err)continue}if result.Gain != tc.expected {t.Errorf("Gain incorrect for %s: got %s, want %s", tc.gainStr, result.Gain, tc.expected)}}})t.Run("should handle negative temperatures", func(t *testing.T) {comment := "Recorded at 21:00:00 24/02/2025 (UTC+13) by AudioMoth 248AB50153AB0549 at medium gain while battery was 4.3V and temperature was -5.2C."
for _, tc := range testCases {comment := "Recorded at 21:00:00 24/02/2025 (UTC+13) by AudioMoth 248AB50153AB0549 at " + tc.gainStr + " gain while battery was 4.3V and temperature was 15.8C."
t.Run("should fallback to legacy parsing", func(t *testing.T) {// Legacy format might not match structured regex but should be parseable// Test with a legacy-style commentcomment := "Recorded at 21:00:00 24/02/2025 (UTC+13) by AudioMoth 248AB50153AB0549 at medium gain while battery was 4.3V and temperature was 15.8C"
func testParseNegativeTemp(t *testing.T) {comment := "Recorded at 21:00:00 24/02/2025 (UTC+13) by AudioMoth 248AB50153AB0549 at medium gain while battery was 4.3V and temperature was -5.2C."result, err := ParseAudioMothComment(comment)if err != nil {t.Fatalf("Failed to parse comment: %v", err)}if result.TempC != -5.2 {t.Errorf("TempC incorrect: got %f, want -5.2", result.TempC)}}
// Note: The legacy parser expects the exact structure, so this might fail// if the comment doesn't match. Adjust test as needed based on actual legacy format.result, err := ParseAudioMothComment(comment)
func testParseLegacyFallback(t *testing.T) {comment := "Recorded at 21:00:00 24/02/2025 (UTC+13) by AudioMoth 248AB50153AB0549 at medium gain while battery was 4.3V and temperature was 15.8C"
// Either succeeds or fails gracefullyif err == nil {// If it succeeds, verify basic fieldsif result.RecorderID == "" {t.Error("RecorderID should not be empty")}
result, err := ParseAudioMothComment(comment)if err == nil {if result.RecorderID == "" {t.Error("RecorderID should not be empty")
// Navigation and editing keys (check by code, not string)
// Printable ASCII character (space handled above via KeySpace)s := msg.String()if len(s) == 1 && s[0] >= 33 && s[0] <= 126 {if len(m.commentText) < 140 {m.commentText = m.commentText[:m.commentCursor] + s + m.commentText[m.commentCursor:]m.commentCursor++}}return m, nil}// handleCommentKeyCode handles navigation and editing keys in comment mode.// Returns true if the key was consumed.func (m *Model) handleCommentKeyCode(key tea.Key) bool {
// Printable ASCII character (space handled above via KeySpace)s := msg.String()if len(s) == 1 && s[0] >= 33 && s[0] <= 126 { // 33='!', 126='~' (space=32 handled above)if len(m.commentText) < 140 {m.commentText = m.commentText[:m.commentCursor] + s + m.commentText[m.commentCursor:]m.commentCursor++}return m, nil}return m, nil
return false
func updateDataset(ctx context.Context, input DatasetInput) (DatasetOutput, error) {var output DatasetOutputdatasetID := *input.ID// Validate ID formatif err := utils.ValidateShortID(datasetID, "dataset_id"); err != nil {return output, err
// validateUpdateInput validates all fields in a dataset update input.func validateUpdateInput(input DatasetInput) error {if err := utils.ValidateShortID(*input.ID, "dataset_id"); err != nil {return err
if input.Type != nil {typeValue := strings.ToLower(*input.Type)if typeValue != "structured" && typeValue != "unstructured" && typeValue != "test" && typeValue != "train" {return output, fmt.Errorf("invalid dataset type: %s (must be 'structured', 'unstructured', 'test', or 'train')", *input.Type)}
if err := validateDatasetType(input.Type); err != nil {return err
// Open writable databasedatabase, err := db.OpenWriteableDB(dbPath)if err != nil {return output, fmt.Errorf("failed to open database: %w", err)}defer database.Close()// Verify dataset exists and check active statusvar exists, active boolerr = database.QueryRow("SELECT EXISTS(SELECT 1 FROM dataset WHERE id = ?), COALESCE((SELECT active FROM dataset WHERE id = ?), false)", datasetID, datasetID).Scan(&exists, &active)if err != nil {return output, fmt.Errorf("failed to query dataset: %w", err)}if !exists {return output, fmt.Errorf("dataset not found: %s", datasetID)
// validateDatasetType validates the type field if provided.func validateDatasetType(t *string) error {if t == nil {return nil
if !active {return output, fmt.Errorf("dataset '%s' is not active (cannot update inactive datasets)", datasetID)
typeValue := strings.ToLower(*t)switch typeValue {case "structured", "unstructured", "test", "train":return nildefault:return fmt.Errorf("invalid dataset type: %s (must be 'structured', 'unstructured', 'test', or 'train')", *t)
func updateDataset(ctx context.Context, input DatasetInput) (DatasetOutput, error) {var output DatasetOutputdatasetID := *input.ID// Validate all fieldsif err := validateUpdateInput(input); err != nil {return output, err}// Open writable databasedatabase, err := db.OpenWriteableDB(dbPath)if err != nil {return output, fmt.Errorf("failed to open database: %w", err)}defer database.Close()// Verify dataset exists and check active statusif err := verifyDatasetActive(database, datasetID); err != nil {return output, err}// Build dynamic UPDATE queryquery, args, err := buildUpdateQuery(input, datasetID)if err != nil {return output, err}
}// verifyDatasetActive checks that a dataset exists and is active.func verifyDatasetActive(database *sql.DB, datasetID string) error {var exists, active boolerr := database.QueryRow("SELECT EXISTS(SELECT 1 FROM dataset WHERE id = ?), COALESCE((SELECT active FROM dataset WHERE id = ?), false)", datasetID, datasetID).Scan(&exists, &active)if err != nil {return fmt.Errorf("failed to query dataset: %w", err)}if !exists {return fmt.Errorf("dataset not found: %s", datasetID)}if !active {return fmt.Errorf("dataset '%s' is not active (cannot update inactive datasets)", datasetID)}return nil
if input.File == "" && input.Folder == "" {output.Errors = append(output.Errors, "either --file or --folder is required")return output, fmt.Errorf("missing required flag: --file or --folder")}if input.Output == "" {output.Errors = append(output.Errors, "--output is required")return output, fmt.Errorf("missing required flag: --output")}if input.Prefix == "" {output.Errors = append(output.Errors, "--prefix is required")return output, fmt.Errorf("missing required flag: --prefix")
if err := validateClipInput(&output, input); err != nil {return output, err
var filePaths []stringvar err errorif input.File != "" {filePaths = []string{input.File}} else {filePaths, err = utils.FindDataFiles(input.Folder)if err != nil {output.Errors = append(output.Errors, fmt.Sprintf("failed to find .data files: %v", err))return output, err}
filePaths, err := resolveClipFiles(&output, input)if err != nil {return output, err
// Sequential for small batchesfor _, dataPath := range filePaths {clips, skipped, errs := processFile(dataPath, input.Output, input.Prefix, input.Filter, speciesName, callType, input.Certainty, imgSize, input.Color, input.WavOnly, input.Night, input.Day, input.Lat, input.Lng, input.Timezone)output.SegmentsClipped += len(clips)if input.Night {output.NightSkipped += skipped} else {output.DaySkipped += skipped}output.OutputFiles = append(output.OutputFiles, clips...)output.Errors = append(output.Errors, errs...)if len(clips) > 0 || len(errs) == 0 {output.FilesProcessed++}}
processFilesSequential(&output, filePaths, input, speciesName, callType, imgSize)
// Parallel file processingtype fileResult struct {clips []stringskipped interrs []string}
processFilesParallel(&output, filePaths, input, speciesName, callType, imgSize)}return output, nil}// validateClipInput validates required flags for clip generation.func validateClipInput(output *CallsClipOutput, input CallsClipInput) error {if input.File == "" && input.Folder == "" {output.Errors = append(output.Errors, "either --file or --folder is required")return fmt.Errorf("missing required flag: --file or --folder")}if input.Output == "" {output.Errors = append(output.Errors, "--output is required")return fmt.Errorf("missing required flag: --output")}if input.Prefix == "" {output.Errors = append(output.Errors, "--prefix is required")return fmt.Errorf("missing required flag: --prefix")}return nil}
workers := min(runtime.NumCPU(), 8, len(filePaths))jobs := make(chan string, len(filePaths))results := make(chan fileResult, len(filePaths))
// resolveClipFiles returns the list of .data file paths from input.func resolveClipFiles(output *CallsClipOutput, input CallsClipInput) ([]string, error) {if input.File != "" {return []string{input.File}, nil}filePaths, err := utils.FindDataFiles(input.Folder)if err != nil {output.Errors = append(output.Errors, fmt.Sprintf("failed to find .data files: %v", err))return nil, err}if len(filePaths) == 0 {output.Errors = append(output.Errors, "no .data files found")return nil, fmt.Errorf("no .data files found")}return filePaths, nil}
var wg sync.WaitGroupfor range workers {wg.Go(func() {for dataPath := range jobs {clips, skipped, errs := processFile(dataPath, input.Output, input.Prefix, input.Filter, speciesName, callType, input.Certainty, imgSize, input.Color, input.WavOnly, input.Night, input.Day, input.Lat, input.Lng, input.Timezone)results <- fileResult{clips: clips, skipped: skipped, errs: errs}}})}
// processFilesSequential processes .data files one at a time.func processFilesSequential(output *CallsClipOutput, filePaths []string, input CallsClipInput, speciesName, callType string, imgSize int) {for _, dataPath := range filePaths {clips, skipped, errs := processFile(dataPath, input.Output, input.Prefix, input.Filter, speciesName, callType, input.Certainty, imgSize, input.Color, input.WavOnly, input.Night, input.Day, input.Lat, input.Lng, input.Timezone)accumulateFileResult(output, clips, skipped, errs, input.Night)}}
for _, dataPath := range filePaths {jobs <- dataPath}close(jobs)
// processFilesParallel processes .data files using worker goroutines.func processFilesParallel(output *CallsClipOutput, filePaths []string, input CallsClipInput, speciesName, callType string, imgSize int) {type fileResult struct {clips []stringskipped interrs []string}
for r := range results {output.SegmentsClipped += len(r.clips)if input.Night {output.NightSkipped += r.skipped} else {output.DaySkipped += r.skipped}output.OutputFiles = append(output.OutputFiles, r.clips...)output.Errors = append(output.Errors, r.errs...)if len(r.clips) > 0 || len(r.errs) == 0 {output.FilesProcessed++
var wg sync.WaitGroupfor range workers {wg.Go(func() {for dataPath := range jobs {clips, skipped, errs := processFile(dataPath, input.Output, input.Prefix, input.Filter, speciesName, callType, input.Certainty, imgSize, input.Color, input.WavOnly, input.Night, input.Day, input.Lat, input.Lng, input.Timezone)results <- fileResult{clips: clips, skipped: skipped, errs: errs}
return output, nil
go func() {wg.Wait()close(results)}()for r := range results {accumulateFileResult(output, r.clips, r.skipped, r.errs, input.Night)}}// accumulateFileResult merges a single file's results into the output.func accumulateFileResult(output *CallsClipOutput, clips []string, skipped int, errs []string, night bool) {output.SegmentsClipped += len(clips)if night {output.NightSkipped += skipped} else {output.DaySkipped += skipped}output.OutputFiles = append(output.OutputFiles, clips...)output.Errors = append(output.Errors, errs...)if len(clips) > 0 || len(errs) == 0 {output.FilesProcessed++}
var matchingSegments []*utils.Segmentfor _, seg := range dataFile.Segments {if seg.SegmentMatchesFilters(filter, speciesName, callType, certainty) {matchingSegments = append(matchingSegments, seg)}}
matchingSegments := filterSegments(dataFile.Segments, filter, speciesName, callType, certainty)
result, err := IsNight(IsNightInput{FilePath: wavPath,Lat: lat,Lng: lng,Timezone: timezone,})if err != nil {fmt.Fprintf(os.Stderr, "warning: skipping %s (isnight error: %v)\n", wavPath, err)
skipped, err := checkDayNightFilter(wavPath, night, day, lat, lng, timezone)if err != nil || skipped {if skipped {return nil, 1, nil}
// Process matching segments (parallel for larger batches)if len(matchingSegments) <= 2 {for _, seg := range matchingSegments {
// Process matching segmentsclips, errors = processSegments(matchingSegments, dataPath, samples, sampleRate, outputDir, prefix, basename, imgSize, color, wavOnly)return clips, 0, errors}// filterSegments returns segments matching the given filter criteria.func filterSegments(segments []*utils.Segment, filter, speciesName, callType string, certainty int) []*utils.Segment {var matching []*utils.Segmentfor _, seg := range segments {if seg.SegmentMatchesFilters(filter, speciesName, callType, certainty) {matching = append(matching, seg)}}return matching}// checkDayNightFilter applies day/night filtering. Returns (skipped=true, nil) if the// recording should be skipped, (false, nil) if it passes, or (false, err) on failure.func checkDayNightFilter(wavPath string, night, day bool, lat, lng float64, timezone string) (bool, error) {result, err := IsNight(IsNightInput{FilePath: wavPath,Lat: lat,Lng: lng,Timezone: timezone,})if err != nil {fmt.Fprintf(os.Stderr, "warning: skipping %s (isnight error: %v)\n", wavPath, err)return false, err}if night && !result.SolarNight {fmt.Fprintf(os.Stderr, "skipped (daytime): %s\n", wavPath)return true, nil}if day && !result.DiurnalActive {fmt.Fprintf(os.Stderr, "skipped (nighttime): %s\n", wavPath)return true, nil}return false, nil}// processSegments generates clips for matching segments, using parallel processing for larger batches.func processSegments(segments []*utils.Segment, dataPath string, samples []float64, sampleRate int, outputDir, prefix, basename string, imgSize int, color, wavOnly bool) ([]string, []string) {var clips []stringvar errors []stringif len(segments) <= 2 {for _, seg := range segments {
var wg sync.WaitGroupfor range workers {wg.Go(func() {for seg := range jobs {clipFiles, err := generateClip(samples, sampleRate, outputDir, prefix, basename, seg.StartTime, seg.EndTime, imgSize, color, wavOnly)if err != nil {results <- segResult{err: fmt.Sprintf("%s: segment %.0f-%.0f: %v", dataPath, seg.StartTime, seg.EndTime, err)}} else {results <- segResult{clips: clipFiles}}
// processSegmentsParallel generates clips for segments using worker goroutines.func processSegmentsParallel(segments []*utils.Segment, dataPath string, samples []float64, sampleRate int, outputDir, prefix, basename string, imgSize int, color, wavOnly bool) ([]string, []string) {type segResult struct {clips []stringerr string}workers := min(runtime.NumCPU(), len(segments))jobs := make(chan *utils.Segment, len(segments))results := make(chan segResult, len(segments))var wg sync.WaitGroupfor range workers {wg.Go(func() {for seg := range jobs {clipFiles, err := generateClip(samples, sampleRate, outputDir, prefix, basename, seg.StartTime, seg.EndTime, imgSize, color, wavOnly)if err != nil {results <- segResult{err: fmt.Sprintf("%s: segment %.0f-%.0f: %v", dataPath, seg.StartTime, seg.EndTime, err)}} else {results <- segResult{clips: clipFiles}
for r := range results {if r.err != "" {errors = append(errors, r.err)} else {clips = append(clips, r.clips...)}
var clips []stringvar errors []stringfor r := range results {if r.err != "" {errors = append(errors, r.err)} else {clips = append(clips, r.clips...)