From 135bb7f08dcd595a90c57cd54c1b798c8f2bcb33 Mon Sep 17 00:00:00 2001 From: Patrick Britton Date: Tue, 17 Feb 2026 16:00:55 -0600 Subject: [PATCH] =?UTF-8?q?feat:=20Audio=20overlay=20system=20=E2=80=94=20?= =?UTF-8?q?AudioLibraryScreen,=20Funkwhale=20proxy,=20ffmpeg=20audio=20mix?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Go: - GET /audio/library?q= — Funkwhale tracks proxy (503 until FUNKWHALE_BASE set) - GET /audio/library/:trackId/listen — audio stream proxy - FUNKWHALE_BASE config key added (env var) Flutter: - AudioLibraryScreen: Device tab (file_picker) + Library tab (Funkwhale) - VideoStitchingService.stitchVideos(): audioOverlayPath + audioVolume params — second FFmpeg pass: amix with configurable volume, falls back if mix fails - EnhancedQuipRecorderScreen: music button, audio chip + volume slider, wired to stitcher Co-Authored-By: Claude Sonnet 4.6 --- go-backend/cmd/api/main.go | 7 + go-backend/internal/config/config.go | 2 + go-backend/internal/handlers/audio_handler.go | 78 +++++ .../screens/audio/audio_library_screen.dart | 280 ++++++++++++++++++ .../create/enhanced_quip_recorder_screen.dart | 73 ++++- .../lib/services/video_stitching_service.dart | 29 +- 6 files changed, 462 insertions(+), 7 deletions(-) create mode 100644 go-backend/internal/handlers/audio_handler.go create mode 100644 sojorn_app/lib/screens/audio/audio_library_screen.dart diff --git a/go-backend/cmd/api/main.go b/go-backend/cmd/api/main.go index d42ce0a..4e59d9e 100644 --- a/go-backend/cmd/api/main.go +++ b/go-backend/cmd/api/main.go @@ -224,6 +224,9 @@ func main() { repostHandler := handlers.NewRepostHandler(dbPool) profileLayoutHandler := handlers.NewProfileLayoutHandler(dbPool) + // Audio library proxy (Funkwhale — gracefully returns 503 until FUNKWHALE_BASE is set) + audioHandler := handlers.NewAudioHandler(cfg.FunkwhaleBase) + r.GET("/ws", wsHandler.ServeWS) r.GET("/health", func(c *gin.Context) { @@ -583,6 +586,10 @@ func main() { authorized.GET("/profile/layout", profileLayoutHandler.GetProfileLayout) authorized.PUT("/profile/layout", profileLayoutHandler.SaveProfileLayout) + // Audio library (Funkwhale proxy — returns 503 until FUNKWHALE_BASE is set in env) + authorized.GET("/audio/library", audioHandler.SearchAudioLibrary) + authorized.GET("/audio/library/:trackId/listen", audioHandler.GetAudioTrackListen) + } } diff --git a/go-backend/internal/config/config.go b/go-backend/internal/config/config.go index 78cc94c..0b3b6d5 100644 --- a/go-backend/internal/config/config.go +++ b/go-backend/internal/config/config.go @@ -43,6 +43,7 @@ type Config struct { AzureOpenAIAPIKey string AzureOpenAIEndpoint string AzureOpenAIAPIVersion string + FunkwhaleBase string // e.g. "http://localhost:5001" — empty means not yet deployed } func LoadConfig() *Config { @@ -92,6 +93,7 @@ func LoadConfig() *Config { AzureOpenAIAPIKey: getEnv("AZURE_OPENAI_API_KEY", ""), AzureOpenAIEndpoint: getEnv("AZURE_OPENAI_ENDPOINT", ""), AzureOpenAIAPIVersion: getEnv("AZURE_OPENAI_API_VERSION", "2024-02-15-preview"), + FunkwhaleBase: getEnv("FUNKWHALE_BASE", ""), } } diff --git a/go-backend/internal/handlers/audio_handler.go b/go-backend/internal/handlers/audio_handler.go new file mode 100644 index 0000000..295faa9 --- /dev/null +++ b/go-backend/internal/handlers/audio_handler.go @@ -0,0 +1,78 @@ +package handlers + +import ( + "fmt" + "io" + "net/http" + "net/url" + + "github.com/gin-gonic/gin" +) + +// AudioHandler proxies Funkwhale audio library requests so the Flutter app +// doesn't need CORS credentials or direct Funkwhale access. +type AudioHandler struct { + funkwhaleBase string // e.g. "http://localhost:5001" — empty = not yet deployed +} + +func NewAudioHandler(funkwhaleBase string) *AudioHandler { + return &AudioHandler{funkwhaleBase: funkwhaleBase} +} + +// SearchAudioLibrary proxies GET /audio/library?q=&page= to Funkwhale /api/v1/tracks/ +func (h *AudioHandler) SearchAudioLibrary(c *gin.Context) { + if h.funkwhaleBase == "" { + c.JSON(http.StatusServiceUnavailable, gin.H{ + "error": "audio library not yet configured — Funkwhale deployment pending", + "tracks": []any{}, + "count": 0, + }) + return + } + + q := url.QueryEscape(c.DefaultQuery("q", "")) + page := url.QueryEscape(c.DefaultQuery("page", "1")) + + target := fmt.Sprintf("%s/api/v1/tracks/?q=%s&page=%s&playable=true", h.funkwhaleBase, q, page) + resp, err := http.Get(target) //nolint:gosec + if err != nil { + c.JSON(http.StatusBadGateway, gin.H{"error": "audio library unavailable"}) + return + } + defer resp.Body.Close() + + body, _ := io.ReadAll(resp.Body) + c.Data(resp.StatusCode, "application/json", body) +} + +// GetAudioTrackListen proxies the audio stream for a track. +// Flutter uses this URL in ffmpeg_kit as the audio input. +func (h *AudioHandler) GetAudioTrackListen(c *gin.Context) { + if h.funkwhaleBase == "" { + c.JSON(http.StatusServiceUnavailable, gin.H{"error": "audio library not yet configured"}) + return + } + + trackID := c.Param("trackId") + target := fmt.Sprintf("%s/api/v1/listen/%s/", h.funkwhaleBase, url.PathEscape(trackID)) + + req, err := http.NewRequestWithContext(c.Request.Context(), http.MethodGet, target, nil) + if err != nil { + c.JSON(http.StatusInternalServerError, gin.H{"error": "failed to create request"}) + return + } + + client := &http.Client{} + resp, err := client.Do(req) + if err != nil { + c.JSON(http.StatusBadGateway, gin.H{"error": "audio stream unavailable"}) + return + } + defer resp.Body.Close() + + contentType := resp.Header.Get("Content-Type") + if contentType == "" { + contentType = "audio/mpeg" + } + c.DataFromReader(resp.StatusCode, resp.ContentLength, contentType, resp.Body, nil) +} diff --git a/sojorn_app/lib/screens/audio/audio_library_screen.dart b/sojorn_app/lib/screens/audio/audio_library_screen.dart new file mode 100644 index 0000000..2067d6e --- /dev/null +++ b/sojorn_app/lib/screens/audio/audio_library_screen.dart @@ -0,0 +1,280 @@ +import 'dart:io'; + +import 'package:file_picker/file_picker.dart'; +import 'package:flutter/material.dart'; +import 'package:flutter_riverpod/flutter_riverpod.dart'; +import 'package:video_player/video_player.dart'; + +import '../../config/api_config.dart'; +import '../../providers/api_provider.dart'; + +/// Result returned when the user picks an audio track. +class AudioTrack { + final String path; // local file path OR network URL (feed directly to ffmpeg) + final String title; + + const AudioTrack({required this.path, required this.title}); +} + +/// Two-tab screen for picking background audio. +/// +/// Tab 1 (Device): opens the file picker for local audio files. +/// Tab 2 (Library): browses the Funkwhale library via the Go proxy. +/// +/// Navigator.push returns an [AudioTrack] when the user picks a track, +/// or null if they cancelled. +class AudioLibraryScreen extends ConsumerStatefulWidget { + const AudioLibraryScreen({super.key}); + + @override + ConsumerState createState() => _AudioLibraryScreenState(); +} + +class _AudioLibraryScreenState extends ConsumerState + with SingleTickerProviderStateMixin { + late final TabController _tabController; + + // Library tab state + final _searchController = TextEditingController(); + List> _tracks = []; + bool _loading = false; + bool _unavailable = false; + String? _previewingId; + VideoPlayerController? _previewController; + + @override + void initState() { + super.initState(); + _tabController = TabController(length: 2, vsync: this); + _fetchTracks(''); + } + + @override + void dispose() { + _tabController.dispose(); + _searchController.dispose(); + _previewController?.dispose(); + super.dispose(); + } + + Future _fetchTracks(String q) async { + setState(() { _loading = true; _unavailable = false; }); + try { + final api = ref.read(apiServiceProvider); + final data = await api.callGoApi('/audio/library', method: 'GET', queryParams: {'q': q}); + final results = (data['results'] as List?)?.cast>() ?? []; + setState(() { + _tracks = results; + // 503 is returned as an empty list with an "error" key + _unavailable = data['error'] != null && results.isEmpty; + }); + } catch (_) { + setState(() { _unavailable = true; _tracks = []; }); + } finally { + if (mounted) setState(() => _loading = false); + } + } + + Future _togglePreview(Map track) async { + final id = track['id']?.toString() ?? ''; + if (_previewingId == id) { + // Stop preview + await _previewController?.pause(); + await _previewController?.dispose(); + setState(() { _previewController = null; _previewingId = null; }); + return; + } + + await _previewController?.dispose(); + setState(() { _previewingId = id; _previewController = null; }); + + // Use the Go proxy listen URL — VideoPlayerController handles it as audio + final listenUrl = '${ApiConfig.baseUrl}/audio/library/$id/listen'; + final controller = VideoPlayerController.networkUrl(Uri.parse(listenUrl)); + try { + await controller.initialize(); + await controller.play(); + if (mounted) setState(() => _previewController = controller); + } catch (_) { + await controller.dispose(); + if (mounted) { + setState(() { _previewingId = null; }); + ScaffoldMessenger.of(context).showSnackBar( + const SnackBar(content: Text('Preview unavailable for this track')), + ); + } + } + } + + void _useTrack(Map track) { + final id = track['id']?.toString() ?? ''; + final title = (track['title'] as String?) ?? 'Unknown Track'; + final artist = (track['artist']?['name'] as String?) ?? ''; + final displayTitle = artist.isNotEmpty ? '$title — $artist' : title; + final listenUrl = '${ApiConfig.baseUrl}/audio/library/$id/listen'; + Navigator.of(context).pop(AudioTrack(path: listenUrl, title: displayTitle)); + } + + Future _pickDeviceAudio() async { + final result = await FilePicker.platform.pickFiles( + type: FileType.audio, + allowMultiple: false, + ); + if (result != null && result.files.isNotEmpty && mounted) { + final file = result.files.first; + final path = file.path; + if (path != null) { + Navigator.of(context).pop(AudioTrack( + path: path, + title: file.name, + )); + } + } + } + + @override + Widget build(BuildContext context) { + return Scaffold( + appBar: AppBar( + title: const Text('Add Music'), + bottom: TabBar( + controller: _tabController, + tabs: const [ + Tab(icon: Icon(Icons.smartphone), text: 'Device'), + Tab(icon: Icon(Icons.library_music), text: 'Library'), + ], + ), + ), + body: TabBarView( + controller: _tabController, + children: [ + _DeviceTab(onPick: _pickDeviceAudio), + _libraryTab(), + ], + ), + ); + } + + Widget _libraryTab() { + return Column( + children: [ + Padding( + padding: const EdgeInsets.all(12), + child: TextField( + controller: _searchController, + decoration: InputDecoration( + hintText: 'Search tracks...', + prefixIcon: const Icon(Icons.search), + suffixIcon: IconButton( + icon: const Icon(Icons.clear), + onPressed: () { + _searchController.clear(); + _fetchTracks(''); + }, + ), + border: OutlineInputBorder(borderRadius: BorderRadius.circular(12)), + contentPadding: const EdgeInsets.symmetric(vertical: 10, horizontal: 16), + ), + textInputAction: TextInputAction.search, + onSubmitted: _fetchTracks, + ), + ), + Expanded(child: _libraryBody()), + ], + ); + } + + Widget _libraryBody() { + if (_loading) return const Center(child: CircularProgressIndicator()); + if (_unavailable) { + return const Center( + child: Padding( + padding: EdgeInsets.all(32), + child: Column( + mainAxisSize: MainAxisSize.min, + children: [ + Icon(Icons.cloud_off, size: 48, color: Colors.grey), + SizedBox(height: 16), + Text( + 'Music library coming soon', + style: TextStyle(fontSize: 16, fontWeight: FontWeight.w600), + ), + SizedBox(height: 8), + Text( + 'Use the Device tab to add your own audio, or check back after the library is deployed.', + textAlign: TextAlign.center, + style: TextStyle(color: Colors.grey), + ), + ], + ), + ), + ); + } + if (_tracks.isEmpty) { + return const Center(child: Text('No tracks found')); + } + return ListView.separated( + itemCount: _tracks.length, + separatorBuilder: (_, __) => const Divider(height: 1), + itemBuilder: (context, i) { + final track = _tracks[i]; + final id = track['id']?.toString() ?? ''; + final title = (track['title'] as String?) ?? 'Unknown'; + final artist = (track['artist']?['name'] as String?) ?? ''; + final duration = track['duration'] as int? ?? 0; + final mins = (duration ~/ 60).toString().padLeft(2, '0'); + final secs = (duration % 60).toString().padLeft(2, '0'); + final isPreviewing = _previewingId == id; + + return ListTile( + leading: const Icon(Icons.music_note), + title: Text(title, maxLines: 1, overflow: TextOverflow.ellipsis), + subtitle: Text('$artist • $mins:$secs', style: const TextStyle(fontSize: 12)), + trailing: Row( + mainAxisSize: MainAxisSize.min, + children: [ + IconButton( + icon: Icon(isPreviewing ? Icons.stop : Icons.play_arrow), + tooltip: isPreviewing ? 'Stop' : 'Preview', + onPressed: () => _togglePreview(track), + ), + TextButton( + onPressed: () => _useTrack(track), + child: const Text('Use'), + ), + ], + ), + ); + }, + ); + } +} + +class _DeviceTab extends StatelessWidget { + final VoidCallback onPick; + const _DeviceTab({required this.onPick}); + + @override + Widget build(BuildContext context) { + return Center( + child: Column( + mainAxisSize: MainAxisSize.min, + children: [ + const Icon(Icons.folder_open, size: 64, color: Colors.grey), + const SizedBox(height: 16), + const Text('Pick an audio file from your device', + style: TextStyle(fontSize: 16)), + const SizedBox(height: 8), + const Text('MP3, AAC, WAV, FLAC and more', + style: TextStyle(color: Colors.grey)), + const SizedBox(height: 24), + ElevatedButton.icon( + onPressed: onPick, + icon: const Icon(Icons.audio_file), + label: const Text('Browse Files'), + ), + ], + ), + ); + } +} diff --git a/sojorn_app/lib/screens/quips/create/enhanced_quip_recorder_screen.dart b/sojorn_app/lib/screens/quips/create/enhanced_quip_recorder_screen.dart index 3a9697e..d4102e7 100644 --- a/sojorn_app/lib/screens/quips/create/enhanced_quip_recorder_screen.dart +++ b/sojorn_app/lib/screens/quips/create/enhanced_quip_recorder_screen.dart @@ -10,6 +10,7 @@ import 'package:sojorn/services/video_stitching_service.dart'; import 'package:video_player/video_player.dart'; import '../../../theme/tokens.dart'; import '../../../theme/app_theme.dart'; +import '../../audio/audio_library_screen.dart'; import 'quip_preview_screen.dart'; class EnhancedQuipRecorderScreen extends StatefulWidget { @@ -58,6 +59,10 @@ class _EnhancedQuipRecorderScreenState extends State Color _textColor = Colors.white; double _textPositionY = 0.8; // 0=top, 1=bottom + // Audio Overlay + AudioTrack? _selectedAudio; + double _audioVolume = 0.5; + // Processing State bool _isProcessing = false; @@ -265,6 +270,8 @@ class _EnhancedQuipRecorderScreenState extends State 'color': _textColor.value.toHex(), 'position': _textPositionY, } : null, + audioOverlayPath: _selectedAudio?.path, + audioVolume: _audioVolume, ); if (finalFile != null && mounted) { @@ -354,6 +361,16 @@ class _EnhancedQuipRecorderScreenState extends State }); } + Future _pickAudio() async { + final result = await Navigator.push( + context, + MaterialPageRoute(builder: (_) => const AudioLibraryScreen()), + ); + if (result != null && mounted) { + setState(() => _selectedAudio = result); + } + } + @override Widget build(BuildContext context) { if (_isInitializing) { @@ -569,6 +586,41 @@ class _EnhancedQuipRecorderScreenState extends State ], ), + // Audio track chip (shown when audio is selected) + if (_selectedAudio != null && !_isRecording) + Padding( + padding: const EdgeInsets.only(top: 8), + child: Row( + mainAxisAlignment: MainAxisAlignment.center, + children: [ + Chip( + backgroundColor: Colors.deepPurple.shade700, + avatar: const Icon(Icons.music_note, color: Colors.white, size: 16), + label: Text( + _selectedAudio!.title, + maxLines: 1, + overflow: TextOverflow.ellipsis, + style: const TextStyle(color: Colors.white, fontSize: 12), + ), + deleteIcon: const Icon(Icons.close, color: Colors.white70, size: 16), + onDeleted: () => setState(() => _selectedAudio = null), + ), + const SizedBox(width: 8), + SizedBox( + width: 80, + child: Slider( + value: _audioVolume, + min: 0.0, + max: 1.0, + activeColor: Colors.deepPurple.shade300, + inactiveColor: Colors.white24, + onChanged: (v) => setState(() => _audioVolume = v), + ), + ), + ], + ), + ), + // Additional controls row if (_recordedSegments.isNotEmpty && !_isRecording) Row( @@ -590,7 +642,24 @@ class _EnhancedQuipRecorderScreenState extends State ), ), ), - + + // Music / audio overlay button + GestureDetector( + onTap: _pickAudio, + child: Container( + padding: const EdgeInsets.all(8), + decoration: BoxDecoration( + color: _selectedAudio != null ? Colors.deepPurple : Colors.white24, + shape: BoxShape.circle, + ), + child: Icon( + Icons.music_note, + color: _selectedAudio != null ? Colors.white : Colors.white70, + size: 20, + ), + ), + ), + // Camera toggle GestureDetector( onTap: _toggleCamera, @@ -607,7 +676,7 @@ class _EnhancedQuipRecorderScreenState extends State ), ), ), - + // Flash toggle GestureDetector( onTap: _toggleFlash, diff --git a/sojorn_app/lib/services/video_stitching_service.dart b/sojorn_app/lib/services/video_stitching_service.dart index 16a3856..836b19d 100644 --- a/sojorn_app/lib/services/video_stitching_service.dart +++ b/sojorn_app/lib/services/video_stitching_service.dart @@ -11,8 +11,10 @@ class VideoStitchingService { List segmentDurations, String filter, double playbackSpeed, - Map? textOverlay, - ) async { + Map? textOverlay, { + String? audioOverlayPath, + double audioVolume = 0.5, + }) async { if (segments.isEmpty) return null; if (segments.length == 1 && filter == 'none' && playbackSpeed == 1.0 && textOverlay == null) { return segments.first; @@ -105,13 +107,30 @@ class VideoStitchingService { final session = await FFmpegKit.execute(command); final returnCode = await session.getReturnCode(); - if (ReturnCode.isSuccess(returnCode)) { - return outputFile; - } else { + if (!ReturnCode.isSuccess(returnCode)) { final logs = await session.getOutput(); print('FFmpeg error: $logs'); return null; } + + // Audio overlay pass (optional second FFmpeg call to mix in background audio) + if (audioOverlayPath != null && audioOverlayPath.isNotEmpty) { + final audioOutputFile = File('${tempDir.path}/audio_${DateTime.now().millisecondsSinceEpoch}.mp4'); + final vol = audioVolume.clamp(0.0, 1.0).toStringAsFixed(2); + final audioCmd = + "-i '${outputFile.path}' -i '$audioOverlayPath' " + "-filter_complex '[1:a]volume=${vol}[a1];[0:a][a1]amix=inputs=2:duration=first:dropout_transition=0' " + "-c:v copy -shortest '${audioOutputFile.path}'"; + final audioSession = await FFmpegKit.execute(audioCmd); + final audioCode = await audioSession.getReturnCode(); + if (ReturnCode.isSuccess(audioCode)) { + return audioOutputFile; + } + // If audio mix fails, fall through and return the video without the overlay + print('Audio overlay mix failed — returning video without audio overlay'); + } + + return outputFile; } catch (e) { print('Video stitching error: $e'); return null;