diff --git a/.gitmodules b/.gitmodules index 21c8215..e69de29 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +0,0 @@ -[submodule "components/esp32-camera"] - path = components/esp32-camera - url = git@github.com:esp-cpp/esp32-camera diff --git a/components/esp32-camera b/components/esp32-camera deleted file mode 160000 index 0fe33c0..0000000 --- a/components/esp32-camera +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 0fe33c0346b9a348bb68190d2b81e830fc46a58b diff --git a/dependencies.lock b/dependencies.lock index e48d52d..f815b7c 100644 --- a/dependencies.lock +++ b/dependencies.lock @@ -1,6 +1,6 @@ dependencies: espp/adc: - component_hash: 3227da5c700828845c2bacc4ce8e5ef49d2ebfe6db6e9378769ffb976f57b526 + component_hash: 8df5e759a2ce090b1d084c4ee5c4e9ac9a60d48a1736f524c29696bd6e99ee4d dependencies: - name: espp/base_component registry_url: https://components.espressif.com @@ -16,9 +16,9 @@ dependencies: source: registry_url: https://components.espressif.com type: service - version: 1.0.30 + version: 1.1.1 espp/base_component: - component_hash: 4eb622f2705843fc76215d51d834ee7297522e883378e706a998d4fa6e49f231 + component_hash: 753dd0037b7dcccb859480dda36eaf7ca83a3f30c466da7c60b957ac10356c8e dependencies: - name: espp/logger registry_url: https://components.espressif.com @@ -30,9 +30,9 @@ dependencies: source: registry_url: https://components.espressif.com type: service - version: 1.0.30 + version: 1.1.1 espp/base_peripheral: - component_hash: d852ab634677571e03f7ab3e871260601c33d46b00c59c56f1361a764734fff6 + component_hash: a6fc75fd03e90382a54cb87e45e720032fed440468111975fe596eb549f05eb9 dependencies: - name: espp/base_component registry_url: https://components.espressif.com @@ -44,9 +44,9 @@ dependencies: source: registry_url: https://components.espressif.com type: service - version: 1.0.30 + version: 1.1.1 espp/bm8563: - component_hash: 5a8c35de2aef522e0af4fe7e6d26dd640c4928c1aed331a350949233789e2deb + component_hash: a89718cd21cff1819ab66606526b500325cb1dd2eb0a947709df549ed0b4f3f3 dependencies: - name: espp/base_peripheral registry_url: https://components.espressif.com @@ -62,9 +62,9 @@ dependencies: source: registry_url: https://components.espressif.com type: service - version: 1.0.30 + version: 1.1.1 espp/cli: - component_hash: 3578f0bdeef0074284aee15d2dc3f3f6d8a36cd198d0c6e39a8c6cc9ebe3203e + component_hash: 129d81da026d387a5bade014e44184549c9d1af553f9042d74aca68b8a60f79a dependencies: - name: espp/logger registry_url: https://components.espressif.com @@ -76,9 +76,9 @@ dependencies: source: registry_url: https://components.espressif.com type: service - version: 1.0.30 + version: 1.1.1 espp/esp32-timer-cam: - component_hash: c7c2c5a23c3eecff8157aee8f31888720f9f45e542c3a858da8ca39a8fc862e3 + component_hash: 7ccb42130c2f4ce70ea2fbfa26a153859ae96b1eae571dbb10da562c894bee74 dependencies: - name: espp/adc registry_url: https://components.espressif.com @@ -120,9 +120,9 @@ dependencies: type: service targets: - esp32 - version: 1.0.30 + version: 1.1.1 espp/format: - component_hash: a36e56d8620d28997f37a41f005bd0af70ccf025f38320738f148e2c9579f2f1 + component_hash: 306438454fb4391109c50ddb43695b8405bbd339c6e219b83694d29e67e41a04 dependencies: - name: idf require: private @@ -130,9 +130,9 @@ dependencies: source: registry_url: https://components.espressif.com type: service - version: 1.0.30 + version: 1.1.1 espp/i2c: - component_hash: 3a2b9d4724627cacf0fd7311ceeb1a54e52dadb555664fb4d110262bee760730 + component_hash: 38b4cbd740ad89e7d96e61f7420f48b7bd7b69a11ebfc4b66ec1ecabbeb0412b dependencies: - name: espp/base_component registry_url: https://components.espressif.com @@ -152,9 +152,9 @@ dependencies: source: registry_url: https://components.espressif.com type: service - version: 1.0.30 + version: 1.1.1 espp/interrupt: - component_hash: efe025b341ab8c5f6c46b165435270c40917fab4203417c040c93733772de65e + component_hash: d104e57d02c8f9e94709354a388d564ab13d2b958bf81f473ea4e4bf5e0fdc60 dependencies: - name: espp/base_component registry_url: https://components.espressif.com @@ -170,9 +170,9 @@ dependencies: source: registry_url: https://components.espressif.com type: service - version: 1.0.30 + version: 1.1.1 espp/led: - component_hash: 54b46f77c1b0a99abfad0b31c5543587413f0c32f5a86f1a4f24fbe561e3bcb2 + component_hash: 4132466e1180e5d53efab93e1369133ec2ae250d874a45f163e9789566db1e5b dependencies: - name: espp/base_component registry_url: https://components.espressif.com @@ -188,9 +188,9 @@ dependencies: source: registry_url: https://components.espressif.com type: service - version: 1.0.30 + version: 1.1.1 espp/logger: - component_hash: 8e6fc4cb3ff9feabd726a5c6fa6465b3fafc704b7c8b283232f9bf98fa517bd8 + component_hash: 3708fb36c62257ad7f81350e70e86c50ce9f42e40b9b707c6b7e21b55b73daa5 dependencies: - name: espp/format registry_url: https://components.espressif.com @@ -202,9 +202,9 @@ dependencies: source: registry_url: https://components.espressif.com type: service - version: 1.0.30 + version: 1.1.1 espp/math: - component_hash: 9fff8c6bcf2db4a715272a211a7f39738f36cad4f7bba90d03ef86429bce7fd8 + component_hash: 6ebde5c1ed8d311a9d4a1fa1c56f77dec02b7e4c958481106cb70ec8cba371f6 dependencies: - name: espp/format registry_url: https://components.espressif.com @@ -216,9 +216,9 @@ dependencies: source: registry_url: https://components.espressif.com type: service - version: 1.0.30 + version: 1.1.1 espp/monitor: - component_hash: 61ec113c1a946bb6a9ad030f449aed0f632b78187ad1409ab386f853e1dea1a7 + component_hash: b80b8a54941cac69547c34a5c711d56fb32404c1a1dac89e728907096d90a418 dependencies: - name: espp/base_component registry_url: https://components.espressif.com @@ -234,23 +234,23 @@ dependencies: source: registry_url: https://components.espressif.com/ type: service - version: 1.0.30 + version: 1.1.1 espp/nvs: - component_hash: 31c1eb637be8d4ab3b34b4a06b12293b545537352b5711d669402f1d8e8d9685 + component_hash: e06b4a5fc2f6c35cd7689fdded1e33bcaeb43638c6b9e2bc8548b8597b37365b dependencies: + - name: idf + require: private + version: '>=5.0' - name: espp/base_component registry_url: https://components.espressif.com require: private version: '>=1.0' - - name: idf - require: private - version: '>=5.0' source: registry_url: https://components.espressif.com/ type: service - version: 1.0.30 + version: 1.1.1 espp/rtsp: - component_hash: baf4a106182c3a0a6e2251fb6e1f771be88e150bcd8109335a9f09e3728eeaa5 + component_hash: eac0533fc9811b800fd73eb0a936827bb3d5c1c76bc7eedc34e0b784e1107c83 dependencies: - name: espp/base_component registry_url: https://components.espressif.com @@ -270,9 +270,9 @@ dependencies: source: registry_url: https://components.espressif.com/ type: service - version: 1.0.30 + version: 1.1.1 espp/socket: - component_hash: 61ec4c78515c373c6db4ca70c8786990abf4cfaa22f3c075374a22e2c27b7eea + component_hash: b9fa3a75f195eb43c1bb0c52ef7d5dc95dee39d9b681392e5dd9ef75baa1b3cd dependencies: - name: espp/base_component registry_url: https://components.espressif.com @@ -288,9 +288,9 @@ dependencies: source: registry_url: https://components.espressif.com/ type: service - version: 1.0.30 + version: 1.1.1 espp/task: - component_hash: cd70ed978a323cd53b25862fc9b00a01eedaff32fbca8e422ae3ea921591d4c7 + component_hash: 6ae353cd01bf89c9236a06f043ba40ee707ae71d09591c3dbbdf1bd1ae48f550 dependencies: - name: espp/base_component registry_url: https://components.espressif.com @@ -302,9 +302,9 @@ dependencies: source: registry_url: https://components.espressif.com/ type: service - version: 1.0.30 + version: 1.1.1 espp/utils: - component_hash: d5b1c58393f34971d49e1fbd00e4429e6653d10b8ae0dc66587f8521875da723 + component_hash: c392a49fe04498463dcdc9e4d36e91003b68384b334bce486b028138be7a5552 dependencies: - name: idf require: private @@ -312,9 +312,9 @@ dependencies: source: registry_url: https://components.espressif.com type: service - version: 1.0.30 + version: 1.1.1 espp/wifi: - component_hash: 762836329b6aaa19dc343718a460859fd14222c37a676c7bf697128d0d52de97 + component_hash: 3099372d99327cca94b2857731345956fa5796dfd113bd5d14bbae2ef98e71fc dependencies: - name: espp/base_component registry_url: https://components.espressif.com @@ -324,15 +324,43 @@ dependencies: registry_url: https://components.espressif.com require: private version: '>=1.0' + - name: espp/nvs + registry_url: https://components.espressif.com + require: private + version: '>=1.0' - name: idf require: private version: '>=5.0' source: registry_url: https://components.espressif.com/ type: service - version: 1.0.30 + version: 1.1.1 + espressif/esp32-camera: + component_hash: bc9c8a6b51df777a014fa295825b3de5069bc0300c317acff20c97cf4a10ac7d + dependencies: + - name: espressif/esp_jpeg + registry_url: https://components.espressif.com + require: public + version: ^1.3.1 + - name: idf + require: private + version: '>=5.1' + source: + registry_url: https://components.espressif.com/ + type: service + version: 2.1.7 + espressif/esp_jpeg: + component_hash: defb83669293cbf86d0fa86b475ba5517aceed04ed70db435388c151ab37b5d7 + dependencies: + - name: idf + require: private + version: '>=5.0' + source: + registry_url: https://components.espressif.com + type: service + version: 1.3.1 espressif/mdns: - component_hash: 3ec0af5f6bce310512e90f482388d21cc7c0e99668172d2f895356165fc6f7c5 + component_hash: 8bcf12e37c58c1d584aef32a02b92548124c7a3a9fcf548d3235c844a035e0f0 dependencies: - name: idf require: private @@ -340,11 +368,11 @@ dependencies: source: registry_url: https://components.espressif.com/ type: service - version: 1.8.2 + version: 1.11.1 idf: source: type: idf - version: 5.5.1 + version: 6.0.0 direct_dependencies: - espp/esp32-timer-cam - espp/monitor @@ -353,8 +381,9 @@ direct_dependencies: - espp/socket - espp/task - espp/wifi +- espressif/esp32-camera - espressif/mdns - idf -manifest_hash: 16aed063bc123c492341a183a55ed2138ec26f48fc99ba25380455c762a86ee9 +manifest_hash: 54d502f6fbd9916123194044b95f82f75ac69f2fc5feae88612fd6c16af8add7 target: esp32 -version: 2.0.0 +version: 3.0.0 diff --git a/main/idf_component.yml b/main/idf_component.yml index 49898de..f7867c1 100644 --- a/main/idf_component.yml +++ b/main/idf_component.yml @@ -4,10 +4,7 @@ dependencies: idf: version: '>=5.0' espressif/mdns: '>=1.8' - # NOTE: we cannot use the idf component because for anything ESP-IDF >= 5.4 it - # automatically uses the new I2C driver, which espp doesn't support yet. - # - # espressif/esp32-camera: '>=2.0' + espressif/esp32-camera: '>=2.0' espp/rtsp: '>=1.0' espp/wifi: '>=1.0' espp/monitor: '>=1.0' diff --git a/main/main.cpp b/main/main.cpp index 11753de..d8dbc7a 100644 --- a/main/main.cpp +++ b/main/main.cpp @@ -10,8 +10,10 @@ #include "cli.hpp" #include "esp32-timer-cam.hpp" +#include "heap_monitor.hpp" #include "nvs.hpp" #include "task.hpp" +#include "task_monitor.hpp" #include "tcp_socket.hpp" #include "udp_socket.hpp" #include "wifi_sta.hpp" @@ -25,13 +27,25 @@ using namespace std::chrono_literals; static espp::Logger logger({.tag = "Camera Streamer", .level = espp::Logger::Verbosity::INFO}); +namespace { +constexpr auto idle_capture_poll_period = 250ms; +constexpr auto dma_pressure_backoff = 250ms; +constexpr auto target_stream_period = 100ms; +constexpr size_t min_dma_free_bytes_for_streaming = 12 * 1024; +constexpr size_t min_dma_largest_block_for_streaming = 4 * 1024; +} // namespace + std::recursive_mutex server_mutex; std::unique_ptr camera_task; +std::unique_ptr memory_monitor_task; +std::unique_ptr task_monitor; std::shared_ptr rtsp_server; +std::atomic frames_streamed{0}; esp_err_t initialize_camera(void); -void start_rtsp_server(std::string_view server_address, int server_port); -bool camera_task_fn(const std::mutex &m, const std::condition_variable &cv); +bool start_rtsp_server(std::string_view server_address, int server_port); +bool camera_task_fn(std::mutex &m, std::condition_variable &cv); +bool memory_monitor_task_fn(std::mutex &m, std::condition_variable &cv, bool &task_notified); extern "C" void app_main(void) { esp_err_t err; @@ -68,6 +82,19 @@ extern "C" void app_main(void) { logger.error("Could not initialize camera: {} '{}'", err, esp_err_to_name(err)); } + logger.info("Starting memory monitors"); + task_monitor = std::make_unique(espp::TaskMonitor::Config{.period = 30s}); + memory_monitor_task = espp::Task::make_unique(espp::Task::Config{ + .callback = memory_monitor_task_fn, + .task_config = + { + .name = "Memory Monitor", + .stack_size_bytes = 6 * 1024, + }, + .log_level = espp::Logger::Verbosity::WARN, + }); + memory_monitor_task->start(); + // initialize WiFi logger.info("Initializing WiFi"); espp::WifiSta wifi_sta( @@ -98,7 +125,10 @@ extern "C" void app_main(void) { // create the camera and rtsp server, and the cv/m // they'll use to communicate std::lock_guard lock(server_mutex); - start_rtsp_server(server_address, CONFIG_RTSP_SERVER_PORT); + if (!start_rtsp_server(server_address, CONFIG_RTSP_SERVER_PORT)) { + logger.error("RTSP server failed to start, not starting camera task"); + return; + } // initialize the camera logger.info("Creating camera task"); camera_task = espp::Task::make_unique( @@ -107,15 +137,23 @@ extern "C" void app_main(void) { camera_task->start(); }}); + if (esp_wifi_set_ps(WIFI_PS_NONE) != ESP_OK) { + logger.warn("Could not disable WiFi power save; RTSP streaming may hit TX backpressure"); + } else { + logger.info("Disabled WiFi power save for RTSP streaming"); + } + espp::WifiStaMenu sta_menu(wifi_sta); auto root_menu = sta_menu.get(); root_menu->Insert( "memory", [](std::ostream &out) { - out << "Minimum free memory: " << heap_caps_get_minimum_free_size(MALLOC_CAP_DEFAULT) + out << "Frames streamed: " << frames_streamed.load() << std::endl; + out << espp::HeapMonitor::get_table( + {MALLOC_CAP_DEFAULT, MALLOC_CAP_INTERNAL, MALLOC_CAP_SPIRAM, MALLOC_CAP_DMA}) << std::endl; }, - "Display minimum free memory."); + "Display current heap monitor information."); root_menu->Insert( "battery", [](std::ostream &out) { @@ -185,9 +223,10 @@ esp_err_t initialize_camera(void) { .jpeg_quality = 15, // 0-63, for OV series camera sensors, lower number means higher quality .fb_count = 2, // When jpeg mode is used, if fb_count more than one, the driver will work in // continuous mode. + .fb_location = CAMERA_FB_IN_PSRAM, .grab_mode = - CAMERA_GRAB_LATEST // CAMERA_GRAB_WHEN_EMPTY // . Sets when buffers should be filled - }; + CAMERA_GRAB_LATEST, // CAMERA_GRAB_WHEN_EMPTY // . Sets when buffers should be filled + .sccb_i2c_port = I2C_NUM_0}; auto err = esp_camera_init(&camera_config); if (err != ESP_OK) { logger.error("Could not initialize camera: {} '{}'", err, esp_err_to_name(err)); @@ -201,7 +240,7 @@ esp_err_t initialize_camera(void) { return ESP_OK; } -void start_rtsp_server(std::string_view server_address, int server_port) { +bool start_rtsp_server(std::string_view server_address, int server_port) { logger.info("Creating RTSP server at {}:{}", server_address, server_port); std::lock_guard lock(server_mutex); rtsp_server = std::make_shared( @@ -210,14 +249,19 @@ void start_rtsp_server(std::string_view server_address, int server_port) { .path = "mjpeg/1", .log_level = espp::Logger::Verbosity::WARN}); rtsp_server->set_session_log_level(espp::Logger::Verbosity::WARN); - rtsp_server->start(); + if (!rtsp_server->start()) { + logger.error("Failed to start RTSP server on {}:{}", server_address, server_port); + rtsp_server.reset(); + return false; + } // initialize mDNS logger.info("Initializing mDNS"); esp_err_t err = mdns_init(); if (err != ESP_OK) { logger.error("Could not initialize mDNS: {}", err); - return; + rtsp_server.reset(); + return false; } uint8_t mac[6]; @@ -226,23 +270,65 @@ void start_rtsp_server(std::string_view server_address, int server_port) { err = mdns_hostname_set(hostname.c_str()); if (err != ESP_OK) { logger.error("Could not set mDNS hostname: {}", err); - return; + mdns_free(); + rtsp_server.reset(); + return false; } logger.info("mDNS hostname set to '{}'", hostname); err = mdns_instance_name_set("Camera Streamer"); if (err != ESP_OK) { logger.error("Could not set mDNS instance name: {}", err); - return; + mdns_free(); + rtsp_server.reset(); + return false; } err = mdns_service_add("RTSP Server", "_rtsp", "_tcp", server_port, NULL, 0); if (err != ESP_OK) { logger.error("Could not add mDNS service: {}", err); - return; + mdns_free(); + rtsp_server.reset(); + return false; } logger.info("mDNS initialized"); + return true; } -bool camera_task_fn(const std::mutex &m, const std::condition_variable &cv) { +bool camera_task_fn(std::mutex &m, std::condition_variable &cv) { + auto start = std::chrono::high_resolution_clock::now(); + auto wait_until = [&](auto deadline) { + std::unique_lock lk(m); + cv.wait_until(lk, deadline); + }; + + { + std::lock_guard lock(server_mutex); + if (!rtsp_server || !rtsp_server->has_active_sessions()) { + wait_until(start + idle_capture_poll_period); + return false; + } + auto recommended_capture_period = rtsp_server->get_recommended_capture_period(); + auto capture_cooldown = rtsp_server->get_capture_cooldown(); + if (capture_cooldown > 0ms) { + wait_until(start + std::max(recommended_capture_period, capture_cooldown)); + return false; + } + } + + auto dma_free = heap_caps_get_free_size(MALLOC_CAP_DMA); + auto dma_largest = heap_caps_get_largest_free_block(MALLOC_CAP_DMA); + if (dma_free < min_dma_free_bytes_for_streaming || + dma_largest < min_dma_largest_block_for_streaming) { + static auto last_pressure_log = std::chrono::steady_clock::time_point{}; + auto now = std::chrono::steady_clock::now(); + if (now - last_pressure_log >= 2s) { + logger.warn("Pausing capture to recover DMA pressure: free={} B, largest_block={} B", + dma_free, dma_largest); + last_pressure_log = now; + } + wait_until(start + dma_pressure_backoff); + return false; + } + // take image static camera_fb_t *fb = NULL; static size_t _jpg_buf_len; @@ -263,10 +349,44 @@ bool camera_task_fn(const std::mutex &m, const std::condition_variable &cv) { } std::span jpg_buf(_jpg_buf, _jpg_buf_len); - espp::JpegFrame image(jpg_buf); std::lock_guard lock(server_mutex); - rtsp_server->send_frame(image); + rtsp_server->send_frame(jpg_buf); + frames_streamed++; esp_camera_fb_return(fb); + + // sleep for a short period to target ~10 FPS to yield to other tasks. + { + auto capture_period = target_stream_period; + { + std::lock_guard lock(server_mutex); + if (rtsp_server) { + capture_period = std::max(capture_period, rtsp_server->get_recommended_capture_period()); + } + } + wait_until(start + capture_period); + } + return false; }; + +bool memory_monitor_task_fn(std::mutex &m, std::condition_variable &cv, bool &task_notified) { + auto start = std::chrono::high_resolution_clock::now(); + static size_t last_frames_streamed = 0; + auto total_frames = frames_streamed.load(); + auto delta_frames = total_frames - last_frames_streamed; + last_frames_streamed = total_frames; + logger.info("Frames streamed: {} (+{} in last interval)\n{}", total_frames, delta_frames, + espp::HeapMonitor::get_table( + {MALLOC_CAP_DEFAULT, MALLOC_CAP_INTERNAL, MALLOC_CAP_SPIRAM, MALLOC_CAP_DMA})); + { + std::unique_lock lk(m); + auto stop_requested = + cv.wait_until(lk, start + 10s, [&task_notified] { return task_notified; }); + task_notified = false; + if (stop_requested) { + return true; + } + } + return false; +} diff --git a/partitions.csv b/partitions.csv index e799571..4196003 100644 --- a/partitions.csv +++ b/partitions.csv @@ -2,4 +2,4 @@ nvs, data, nvs, 0x9000, 0x6000 phy_init, data, phy, 0xf000, 0x1000 factory, app, factory, 0x10000, 2M -littlefs, data, spiffs, , 1M +littlefs, data, littlefs, , 1M diff --git a/sdkconfig.defaults b/sdkconfig.defaults index bba230d..41dc105 100644 --- a/sdkconfig.defaults +++ b/sdkconfig.defaults @@ -16,6 +16,7 @@ CONFIG_ESP_MAIN_TASK_STACK_SIZE=32768 # SPIRAM Configuration CONFIG_SPIRAM=y CONFIG_SPIRAM_USE_MALLOC=y +CONFIG_SPIRAM_MODE_OCT=y CONFIG_SPIRAM_SPEED_80M=y # CONFIG_SPIRAM_ALLOW_BSS_SEG_EXTERNAL_MEMORY=y @@ -32,7 +33,22 @@ CONFIG_ESP_DEFAULT_CPU_FREQ_MHZ_240=y CONFIG_ESP_DEFAULT_CPU_FREQ_MHZ=240 # ESP32-Camera specific -CONFIG_SCCB_HARDWARE_I2C_DRIVER_LEGACY=y +# CONFIG_SCCB_HARDWARE_I2C_DRIVER_LEGACY=y -# the cli library requires exceptions right now... -CONFIG_COMPILER_CXX_EXCEPTIONS=y +# CONFIG_LWIP_IRAM_OPTIMIZATION=y +CONFIG_LWIP_TCPIP_TASK_PRIO=23 + +# +# ESP32S3-specific +# +CONFIG_ESP_WIFI_STATIC_RX_BUFFER_NUM=8 +CONFIG_ESP_WIFI_DYNAMIC_RX_BUFFER_NUM=16 +CONFIG_ESP_WIFI_STATIC_TX_BUFFER=y +CONFIG_ESP_WIFI_STATIC_TX_BUFFER_NUM=16 +CONFIG_ESP_WIFI_AMPDU_TX_ENABLED=y + +CONFIG_LWIP_TCP_SND_BUF_DEFAULT=16384 +CONFIG_LWIP_TCP_WND_DEFAULT=16384 +CONFIG_LWIP_TCP_RECVMBOX_SIZE=16 +CONFIG_LWIP_UDP_RECVMBOX_SIZE=16 +CONFIG_LWIP_TCPIP_RECVMBOX_SIZE=16