From 1ce24cdf7b498d31e9af9b9e74c192316864f6b3 Mon Sep 17 00:00:00 2001 From: Travis Vasceannie Date: Thu, 15 Jan 2026 15:58:06 +0000 Subject: [PATCH] feat: reorganize Claude hooks and add RAG documentation structure with error handling policies - Moved all hookify configuration files from `.claude/` to `.claude/hooks/` subdirectory for better organization - Added four new blocking hooks to prevent common error handling anti-patterns: - `block-broad-exception-handler`: Prevents catching generic `Exception` with only logging - `block-datetime-now-fallback`: Blocks returning `datetime.now()` as fallback on parse failures to prevent data corruption - `block-default --- ...ify.block-broad-exception-handler.local.md | 61 + ...okify.block-datetime-now-fallback.local.md | 42 + ...okify.block-default-value-swallow.local.md | 60 + .../hookify.block-silent-none-return.local.md | 51 + .../hookify.block-any-type.local.md | 0 .../hookify.block-assertion-roulette.local.md | 0 ...kify.block-code-quality-test-bash.local.md | 0 ...ify.block-code-quality-test-edits.local.md | 0 ...k-code-quality-test-serena-plugin.local.md | 0 ...fy.block-code-quality-test-serena.local.md | 0 .../hookify.block-duplicate-fixtures.local.md | 0 .../hookify.block-ignore-preexisting.local.md | 0 ...block-linter-config-frontend-bash.local.md | 0 ...kify.block-linter-config-frontend.local.md | 0 ...y.block-linter-config-python-bash.local.md | 0 ...ookify.block-linter-config-python.local.md | 0 .../hookify.block-magic-numbers.local.md | 0 .../hookify.block-makefile-bash.local.md | 0 .../hookify.block-makefile-edit.local.md | 0 ...ify.block-test-loops-conditionals.local.md | 0 .../hookify.block-tests-quality-bash.local.md | 0 .../hookify.block-tests-quality.local.md | 0 .../hookify.block-type-ignore.local.md | 0 .../hookify.require-make-quality.local.md | 0 .../hookify.warn-baselines-edit-bash.local.md | 0 .../hookify.warn-baselines-edit.local.md | 0 .../hookify.warn-large-file.local.md | 0 .../hookify.warn-new-file-search.local.md | 0 .mcp.json | 8 + .rag/01-architecture-overview.md | 69 + .rag/02-domain-entities.md | 67 + .rag/03-domain-ports.md | 133 ++ .rag/04-application-services.md | 142 ++ .rag/05-infrastructure-adapters.md | 153 +++ .rag/06-grpc-layer.md | 140 ++ .rag/07-typescript-api-layer.md | 172 +++ .rag/08-typescript-hooks-contexts.md | 212 +++ .rag/09-typescript-components-pages.md | 129 ++ .rag/10-rust-tauri-commands.md | 238 ++++ .rag/11-rust-grpc-types.md | 191 +++ .rag/12-rust-state-audio-crypto.md | 307 +++++ .rag/13-common-utilities.md | 196 +++ CLAUDE.md | 1188 ++++------------- client/CLAUDE.md | 237 +++- client/src/CLAUDE.md | 338 +++++ client/src/api/reconnection.ts | 16 +- .../components/entity-management-panel.tsx | 16 +- .../advanced-local-ai-settings/_constants.ts | 16 +- .../model-auth-section.tsx | 4 +- .../components/settings/ai-config-section.tsx | 4 +- client/src/hooks/use-auth-flow.ts | 8 +- client/src/hooks/use-recording-session.ts | 20 +- .../lib/ai-providers/model-catalog-utils.ts | 4 +- client/src/lib/crypto.ts | 4 +- client/src/lib/log-messages.ts | 2 +- client/src/pages/Settings.tsx | 4 +- docker/CLAUDE.md | 1035 ++++++++++++++ .../sprint-4-ner-extraction/README.md | 2 +- .../sprint-6-webhooks/README.md | 10 +- .../TESTING.md | 2 +- .../.archive/sprint_04_constant_imports.md | 2 +- repomix.config.json | 2 +- scripts/ab_streaming_harness.py | 2 +- src/AGENTS.md | 419 ++++++ src/CLAUDE.md | 419 ++++++ src/noteflow/CLAUDE.md | 608 +++++++++ .../{ports.py => ports/__init__.py} | 0 src/noteflow/application/services/__init__.py | 9 +- .../services/asr_config/__init__.py | 31 + .../persistence.py} | 12 +- .../service.py} | 4 +- .../types.py} | 0 .../application/services/auth/__init__.py | 14 + .../{auth_constants.py => auth/constants.py} | 0 .../integration_manager.py} | 2 +- .../{auth_service.py => auth/service.py} | 13 +- .../token_exchanger.py} | 32 +- .../services/{auth_types.py => auth/types.py} | 17 + .../{auth_workflows.py => auth/workflows.py} | 25 +- .../application/services/export/__init__.py | 10 + .../{export_service.py => export/service.py} | 2 +- .../services/huggingface/__init__.py | 9 + .../service.py} | 0 .../services/identity/identity_service.py | 4 +- .../application/services/ner/__init__.py | 8 + .../{ner_service.py => ner/service.py} | 0 .../{protocols.py => protocols/__init__.py} | 5 + .../services/recovery/_job_recoverer.py | 44 +- .../services/recovery/recovery_service.py | 17 +- .../services/retention/__init__.py | 8 + .../service.py} | 46 +- .../services/streaming_config/__init__.py | 23 + .../persistence.py} | 6 +- .../application/services/triggers/__init__.py | 8 + .../service.py} | 0 .../application/services/webhooks/__init__.py | 7 + .../service.py} | 0 src/noteflow/config/constants/core.py | 3 + src/noteflow/config/settings/_main.py | 3 +- src/noteflow/domain/constants/fields.py | 3 + src/noteflow/domain/entities/meeting.py | 6 +- src/noteflow/grpc/AGENTS.md | 638 +++++++++ src/noteflow/grpc/CLAUDE.md | 638 +++++++++ src/noteflow/grpc/_client_mixins/__init__.py | 20 - src/noteflow/grpc/_types.py | 108 -- src/noteflow/grpc/client.py | 18 +- src/noteflow/grpc/client_mixins/__init__.py | 20 + src/noteflow/grpc/client_mixins/_constants.py | 28 + .../grpc/client_mixins/_error_handling.py | 77 ++ .../annotation.py | 6 +- .../converters.py | 2 +- .../diarization.py | 6 +- .../export.py | 6 +- .../meeting.py | 6 +- .../protocols.py | 4 +- .../streaming.py | 6 +- src/noteflow/grpc/config/__init__.py | 1 + src/noteflow/grpc/{_cli.py => config/cli.py} | 2 +- .../grpc/{_config.py => config/config.py} | 4 +- .../grpc/{_constants.py => constants.py} | 0 src/noteflow/grpc/identity/__init__.py | 1 + .../singleton.py} | 0 src/noteflow/grpc/interceptors/identity.py | 6 - .../grpc/{_mixins => mixins}/__init__.py | 0 .../{_mixins => mixins}/_audio_processing.py | 0 src/noteflow/grpc/mixins/_metrics.py | 85 ++ .../grpc/{_mixins => mixins}/_model_status.py | 13 +- .../_repository_protocols.py | 0 .../{_mixins => mixins}/_servicer_state.py | 10 +- src/noteflow/grpc/mixins/_task_callbacks.py | 78 ++ .../grpc/{_mixins => mixins}/_types.py | 0 .../grpc/{_mixins => mixins}/annotation.py | 0 .../grpc/{_mixins => mixins}/asr_config.py | 2 +- .../grpc/{_mixins => mixins}/calendar.py | 0 .../converters/__init__.py | 0 .../{_mixins => mixins}/converters/_domain.py | 0 .../converters/_external.py | 0 .../converters/_id_parsing.py | 0 .../{_mixins => mixins}/converters/_oidc.py | 0 .../converters/_timestamps.py | 0 .../diarization/__init__.py | 0 .../{_mixins => mixins}/diarization/_jobs.py | 0 .../{_mixins => mixins}/diarization/_mixin.py | 0 .../diarization/_refinement.py | 0 .../diarization/_speaker.py | 0 .../diarization/_status.py | 0 .../diarization/_streaming.py | 0 .../{_mixins => mixins}/diarization/_types.py | 0 .../{_mixins => mixins}/diarization_job.py | 0 .../grpc/{_mixins => mixins}/entities.py | 2 +- .../{_mixins => mixins}/errors/__init__.py | 0 .../grpc/{_mixins => mixins}/errors/_abort.py | 0 .../{_mixins => mixins}/errors/_constants.py | 0 .../grpc/{_mixins => mixins}/errors/_fetch.py | 0 .../grpc/{_mixins => mixins}/errors/_parse.py | 0 .../{_mixins => mixins}/errors/_require.py | 4 +- .../{_mixins => mixins}/errors/_webhooks.py | 0 .../grpc/{_mixins => mixins}/export.py | 2 +- .../grpc/{_mixins => mixins}/hf_token.py | 2 +- .../grpc/{_mixins => mixins}/identity.py | 2 +- .../{_mixins => mixins}/meeting/__init__.py | 0 .../meeting/_post_processing.py | 2 +- .../meeting/_project_scope.py | 8 +- .../{_mixins => mixins}/meeting/_stop_ops.py | 2 +- .../meeting/meeting_mixin.py | 2 +- .../grpc/{_mixins => mixins}/observability.py | 0 .../grpc/{_mixins => mixins}/oidc/__init__.py | 0 .../grpc/{_mixins => mixins}/oidc/_support.py | 0 .../{_mixins => mixins}/oidc/oidc_mixin.py | 0 .../grpc/{_mixins => mixins}/preferences.py | 0 .../{_mixins => mixins}/project/__init__.py | 0 .../project/_converters.py | 0 .../project/_membership.py | 0 .../{_mixins => mixins}/project/_mixin.py | 0 .../grpc/{_mixins => mixins}/protocols.py | 0 .../servicer_core/__init__.py | 0 .../servicer_core/protocols.py | 0 .../servicer_other/__init__.py | 0 .../servicer_other/protocols.py | 2 +- .../{_mixins => mixins}/streaming/__init__.py | 0 .../{_mixins => mixins}/streaming/_asr.py | 0 .../{_mixins => mixins}/streaming/_cleanup.py | 0 .../{_mixins => mixins}/streaming/_mixin.py | 3 +- .../streaming/_partials.py | 0 .../streaming/_processing/__init__.py | 0 .../streaming/_processing/_audio_ops.py | 0 .../streaming/_processing/_chunk_tracking.py | 0 .../streaming/_processing/_congestion.py | 3 +- .../streaming/_processing/_constants.py | 0 .../streaming/_processing/_types.py | 0 .../streaming/_processing/_vad_processing.py | 0 .../{_mixins => mixins}/streaming/_session.py | 7 +- .../streaming/_session_helpers.py | 0 .../{_mixins => mixins}/streaming/_types.py | 0 .../{_mixins => mixins}/streaming_config.py | 13 +- .../summarization/__init__.py | 0 .../summarization/_consent.py | 0 .../summarization/_consent_mixin.py | 0 .../summarization/_constants.py | 0 .../summarization/_context_builders.py | 0 .../summarization/_generation_mixin.py | 4 +- .../summarization/_summary_generation.py | 0 .../summarization/_template_crud.py | 0 .../summarization/_template_resolution.py | 0 .../summarization/_templates_mixin.py | 0 src/noteflow/grpc/{_mixins => mixins}/sync.py | 0 .../grpc/{_mixins => mixins}/webhooks.py | 0 src/noteflow/grpc/proto/noteflow_pb2_grpc.py | 6 +- src/noteflow/grpc/server/__init__.py | 28 +- src/noteflow/grpc/server/health.py | 64 + src/noteflow/grpc/server/internal/__init__.py | 1 + .../{_bootstrap.py => internal/bootstrap.py} | 10 +- .../{_lifecycle.py => internal/lifecycle.py} | 2 +- .../{_services.py => internal/services.py} | 12 +- .../server/{_setup.py => internal/setup.py} | 8 +- .../streaming_config.py} | 2 +- .../server/{_types.py => internal/types.py} | 4 +- src/noteflow/grpc/service.py | 26 +- src/noteflow/grpc/servicer/__init__.py | 1 + .../{_service_base.py => servicer/base.py} | 2 +- .../mixins.py} | 10 +- .../shutdown.py} | 4 +- .../{_service_stubs.py => servicer/stubs.py} | 4 +- src/noteflow/grpc/startup/__init__.py | 1 + .../{_startup_banner.py => startup/banner.py} | 4 +- .../server_bootstrap.py} | 0 .../services.py} | 6 +- .../grpc/{_startup.py => startup/startup.py} | 6 +- src/noteflow/grpc/types/__init__.py | 203 +++ .../infrastructure/audio/partial_buffer.py | 3 + src/noteflow/infrastructure/auth/_presets.py | 3 +- .../infrastructure/auth/oidc_registry.py | 6 +- .../infrastructure/calendar/google_adapter.py | 82 +- .../infrastructure/calendar/oauth_flow.py | 4 +- .../calendar/outlook/_event_fetcher.py | 8 +- .../calendar/outlook/_event_parser.py | 72 +- .../converters/integration_converters.py | 4 +- .../infrastructure/metrics/__init__.py | 10 + .../metrics/infrastructure_metrics.py | 257 ++++ .../observability/usage/_database_sink.py | 4 +- .../persistence/models/_columns.py | 18 +- .../infrastructure/security/keystore.py | 6 +- .../summarization/ollama_provider.py | 12 +- .../summarization/template_renderer.py | 8 +- .../triggers/app_audio/_sampler.py | 17 +- .../infrastructure/triggers/foreground_app.py | 18 +- tests/application/conftest.py | 2 +- tests/application/test_asr_config_service.py | 28 +- tests/application/test_auth_service.py | 8 +- tests/application/test_export_service.py | 2 +- tests/application/test_hf_token_service.py | 10 +- tests/application/test_meeting_service.py | 13 +- tests/application/test_ner_service.py | 18 +- tests/application/test_retention_service.py | 2 +- tests/application/test_trigger_service.py | 2 +- tests/application/test_webhook_service.py | 2 +- tests/domain/conftest.py | 26 + tests/domain/test_annotation.py | 19 +- tests/domain/test_meeting.py | 14 +- tests/domain/test_named_entity.py | 24 +- tests/domain/test_value_objects.py | 2 +- tests/grpc/conftest.py | 39 + tests/grpc/proto_types.py | 2 +- tests/grpc/test_annotation_mixin.py | 4 +- tests/grpc/test_chunk_sequence_tracking.py | 10 +- tests/grpc/test_client_result.py | 215 +++ tests/grpc/test_cloud_consent.py | 2 +- tests/grpc/test_congestion_tracking.py | 19 +- tests/grpc/test_diarization_lifecycle.py | 21 +- tests/grpc/test_diarization_mixin.py | 32 +- tests/grpc/test_diarization_refine.py | 2 +- tests/grpc/test_entities_mixin.py | 10 +- tests/grpc/test_export_mixin.py | 30 +- tests/grpc/test_generate_summary.py | 2 +- tests/grpc/test_identity_mixin.py | 4 +- tests/grpc/test_interceptors.py | 18 +- tests/grpc/test_meeting_mixin.py | 140 +- tests/grpc/test_mixin_helpers.py | 2 +- tests/grpc/test_oauth.py | 2 +- tests/grpc/test_observability_mixin.py | 44 +- tests/grpc/test_oidc_mixin.py | 4 +- tests/grpc/test_partial_transcription.py | 18 +- tests/grpc/test_preferences_mixin.py | 4 +- tests/grpc/test_project_mixin.py | 2 +- tests/grpc/test_proto_compilation.py | 2 +- tests/grpc/test_server_auto_enable.py | 23 +- tests/grpc/test_server_health.py | 383 ++++++ tests/grpc/test_stream_lifecycle.py | 15 +- tests/grpc/test_streaming_metrics.py | 295 ++++ tests/grpc/test_sync_orchestration.py | 2 +- tests/grpc/test_task_callbacks.py | 403 ++++++ tests/grpc/test_timestamp_converters.py | 2 +- tests/grpc/test_webhooks_mixin.py | 14 +- .../infrastructure/auth/test_oidc_registry.py | 8 +- .../metrics/test_infrastructure_metrics.py | 232 ++++ .../observability/test_log_buffer.py | 51 +- .../observability/test_usage.py | 2 +- .../persistence/test_logging_persistence.py | 12 +- .../persistence/test_migrations.py | 4 - .../infrastructure/triggers/test_calendar.py | 46 +- .../triggers/test_foreground_app.py | 10 +- tests/integration/test_e2e_export.py | 2 +- tests/integration/test_e2e_ner.py | 4 +- tests/integration/test_e2e_streaming.py | 4 +- tests/integration/test_e2e_summarization.py | 2 +- .../test_grpc_servicer_database.py | 2 +- tests/integration/test_hf_token_grpc.py | 2 +- tests/integration/test_recovery_service.py | 16 +- tests/integration/test_webhook_integration.py | 4 +- tests/quality/_detectors/__init__.py | 86 ++ tests/quality/_detectors/code_smells.py | 369 +++++ tests/quality/_detectors/stale_code.py | 163 +++ .../test_smells.py} | 218 +-- tests/quality/_detectors/wrappers.py | 230 ++++ tests/quality/baselines.json | 44 +- tests/quality/generate_baseline.py | 687 +--------- tests/quality/test_code_smells.py | 337 +---- tests/quality/test_stale_code.py | 135 +- tests/quality/test_test_smells.py | 686 +--------- tests/quality/test_unnecessary_wrappers.py | 215 +-- tests/test_server_address_sync.py | 2 +- 321 files changed, 11515 insertions(+), 3878 deletions(-) create mode 100644 .claude/hookify.block-broad-exception-handler.local.md create mode 100644 .claude/hookify.block-datetime-now-fallback.local.md create mode 100644 .claude/hookify.block-default-value-swallow.local.md create mode 100644 .claude/hookify.block-silent-none-return.local.md rename .claude/{ => hooks}/hookify.block-any-type.local.md (100%) rename .claude/{ => hooks}/hookify.block-assertion-roulette.local.md (100%) rename .claude/{ => hooks}/hookify.block-code-quality-test-bash.local.md (100%) rename .claude/{ => hooks}/hookify.block-code-quality-test-edits.local.md (100%) rename .claude/{ => hooks}/hookify.block-code-quality-test-serena-plugin.local.md (100%) rename .claude/{ => hooks}/hookify.block-code-quality-test-serena.local.md (100%) rename .claude/{ => hooks}/hookify.block-duplicate-fixtures.local.md (100%) rename .claude/{ => hooks}/hookify.block-ignore-preexisting.local.md (100%) rename .claude/{ => hooks}/hookify.block-linter-config-frontend-bash.local.md (100%) rename .claude/{ => hooks}/hookify.block-linter-config-frontend.local.md (100%) rename .claude/{ => hooks}/hookify.block-linter-config-python-bash.local.md (100%) rename .claude/{ => hooks}/hookify.block-linter-config-python.local.md (100%) rename .claude/{ => hooks}/hookify.block-magic-numbers.local.md (100%) rename .claude/{ => hooks}/hookify.block-makefile-bash.local.md (100%) rename .claude/{ => hooks}/hookify.block-makefile-edit.local.md (100%) rename .claude/{ => hooks}/hookify.block-test-loops-conditionals.local.md (100%) rename .claude/{ => hooks}/hookify.block-tests-quality-bash.local.md (100%) rename .claude/{ => hooks}/hookify.block-tests-quality.local.md (100%) rename .claude/{ => hooks}/hookify.block-type-ignore.local.md (100%) rename .claude/{ => hooks}/hookify.require-make-quality.local.md (100%) rename .claude/{ => hooks}/hookify.warn-baselines-edit-bash.local.md (100%) rename .claude/{ => hooks}/hookify.warn-baselines-edit.local.md (100%) rename .claude/{ => hooks}/hookify.warn-large-file.local.md (100%) rename .claude/{ => hooks}/hookify.warn-new-file-search.local.md (100%) create mode 100644 .mcp.json create mode 100644 .rag/01-architecture-overview.md create mode 100644 .rag/02-domain-entities.md create mode 100644 .rag/03-domain-ports.md create mode 100644 .rag/04-application-services.md create mode 100644 .rag/05-infrastructure-adapters.md create mode 100644 .rag/06-grpc-layer.md create mode 100644 .rag/07-typescript-api-layer.md create mode 100644 .rag/08-typescript-hooks-contexts.md create mode 100644 .rag/09-typescript-components-pages.md create mode 100644 .rag/10-rust-tauri-commands.md create mode 100644 .rag/11-rust-grpc-types.md create mode 100644 .rag/12-rust-state-audio-crypto.md create mode 100644 .rag/13-common-utilities.md create mode 100644 client/src/CLAUDE.md create mode 100644 docker/CLAUDE.md create mode 100644 src/AGENTS.md create mode 100644 src/CLAUDE.md create mode 100644 src/noteflow/CLAUDE.md rename src/noteflow/application/observability/{ports.py => ports/__init__.py} (100%) create mode 100644 src/noteflow/application/services/asr_config/__init__.py rename src/noteflow/application/services/{asr_config_persistence.py => asr_config/persistence.py} (95%) rename src/noteflow/application/services/{asr_config_service.py => asr_config/service.py} (98%) rename src/noteflow/application/services/{asr_config_types.py => asr_config/types.py} (100%) create mode 100644 src/noteflow/application/services/auth/__init__.py rename src/noteflow/application/services/{auth_constants.py => auth/constants.py} (100%) rename src/noteflow/application/services/{auth_integration_manager.py => auth/integration_manager.py} (98%) rename src/noteflow/application/services/{auth_service.py => auth/service.py} (95%) rename src/noteflow/application/services/{auth_token_exchanger.py => auth/token_exchanger.py} (80%) rename src/noteflow/application/services/{auth_types.py => auth/types.py} (86%) rename src/noteflow/application/services/{auth_workflows.py => auth/workflows.py} (89%) create mode 100644 src/noteflow/application/services/export/__init__.py rename src/noteflow/application/services/{export_service.py => export/service.py} (99%) create mode 100644 src/noteflow/application/services/huggingface/__init__.py rename src/noteflow/application/services/{hf_token_service.py => huggingface/service.py} (100%) create mode 100644 src/noteflow/application/services/ner/__init__.py rename src/noteflow/application/services/{ner_service.py => ner/service.py} (100%) rename src/noteflow/application/services/{protocols.py => protocols/__init__.py} (91%) create mode 100644 src/noteflow/application/services/retention/__init__.py rename src/noteflow/application/services/{retention_service.py => retention/service.py} (79%) create mode 100644 src/noteflow/application/services/streaming_config/__init__.py rename src/noteflow/application/services/{streaming_config_persistence.py => streaming_config/persistence.py} (97%) create mode 100644 src/noteflow/application/services/triggers/__init__.py rename src/noteflow/application/services/{trigger_service.py => triggers/service.py} (100%) create mode 100644 src/noteflow/application/services/webhooks/__init__.py rename src/noteflow/application/services/{webhook_service.py => webhooks/service.py} (100%) create mode 100644 src/noteflow/grpc/AGENTS.md create mode 100644 src/noteflow/grpc/CLAUDE.md delete mode 100644 src/noteflow/grpc/_client_mixins/__init__.py delete mode 100644 src/noteflow/grpc/_types.py create mode 100644 src/noteflow/grpc/client_mixins/__init__.py create mode 100644 src/noteflow/grpc/client_mixins/_constants.py create mode 100644 src/noteflow/grpc/client_mixins/_error_handling.py rename src/noteflow/grpc/{_client_mixins => client_mixins}/annotation.py (97%) rename src/noteflow/grpc/{_client_mixins => client_mixins}/converters.py (98%) rename src/noteflow/grpc/{_client_mixins => client_mixins}/diarization.py (95%) rename src/noteflow/grpc/{_client_mixins => client_mixins}/export.py (89%) rename src/noteflow/grpc/{_client_mixins => client_mixins}/meeting.py (95%) rename src/noteflow/grpc/{_client_mixins => client_mixins}/protocols.py (96%) rename src/noteflow/grpc/{_client_mixins => client_mixins}/streaming.py (96%) create mode 100644 src/noteflow/grpc/config/__init__.py rename src/noteflow/grpc/{_cli.py => config/cli.py} (99%) rename src/noteflow/grpc/{_config.py => config/config.py} (97%) rename src/noteflow/grpc/{_constants.py => constants.py} (100%) create mode 100644 src/noteflow/grpc/identity/__init__.py rename src/noteflow/grpc/{_identity_singleton.py => identity/singleton.py} (100%) rename src/noteflow/grpc/{_mixins => mixins}/__init__.py (100%) rename src/noteflow/grpc/{_mixins => mixins}/_audio_processing.py (100%) create mode 100644 src/noteflow/grpc/mixins/_metrics.py rename src/noteflow/grpc/{_mixins => mixins}/_model_status.py (93%) rename src/noteflow/grpc/{_mixins => mixins}/_repository_protocols.py (100%) rename src/noteflow/grpc/{_mixins => mixins}/_servicer_state.py (91%) create mode 100644 src/noteflow/grpc/mixins/_task_callbacks.py rename src/noteflow/grpc/{_mixins => mixins}/_types.py (100%) rename src/noteflow/grpc/{_mixins => mixins}/annotation.py (100%) rename src/noteflow/grpc/{_mixins => mixins}/asr_config.py (99%) rename src/noteflow/grpc/{_mixins => mixins}/calendar.py (100%) rename src/noteflow/grpc/{_mixins => mixins}/converters/__init__.py (100%) rename src/noteflow/grpc/{_mixins => mixins}/converters/_domain.py (100%) rename src/noteflow/grpc/{_mixins => mixins}/converters/_external.py (100%) rename src/noteflow/grpc/{_mixins => mixins}/converters/_id_parsing.py (100%) rename src/noteflow/grpc/{_mixins => mixins}/converters/_oidc.py (100%) rename src/noteflow/grpc/{_mixins => mixins}/converters/_timestamps.py (100%) rename src/noteflow/grpc/{_mixins => mixins}/diarization/__init__.py (100%) rename src/noteflow/grpc/{_mixins => mixins}/diarization/_jobs.py (100%) rename src/noteflow/grpc/{_mixins => mixins}/diarization/_mixin.py (100%) rename src/noteflow/grpc/{_mixins => mixins}/diarization/_refinement.py (100%) rename src/noteflow/grpc/{_mixins => mixins}/diarization/_speaker.py (100%) rename src/noteflow/grpc/{_mixins => mixins}/diarization/_status.py (100%) rename src/noteflow/grpc/{_mixins => mixins}/diarization/_streaming.py (100%) rename src/noteflow/grpc/{_mixins => mixins}/diarization/_types.py (100%) rename src/noteflow/grpc/{_mixins => mixins}/diarization_job.py (100%) rename src/noteflow/grpc/{_mixins => mixins}/entities.py (98%) rename src/noteflow/grpc/{_mixins => mixins}/errors/__init__.py (100%) rename src/noteflow/grpc/{_mixins => mixins}/errors/_abort.py (100%) rename src/noteflow/grpc/{_mixins => mixins}/errors/_constants.py (100%) rename src/noteflow/grpc/{_mixins => mixins}/errors/_fetch.py (100%) rename src/noteflow/grpc/{_mixins => mixins}/errors/_parse.py (100%) rename src/noteflow/grpc/{_mixins => mixins}/errors/_require.py (98%) rename src/noteflow/grpc/{_mixins => mixins}/errors/_webhooks.py (100%) rename src/noteflow/grpc/{_mixins => mixins}/export.py (97%) rename src/noteflow/grpc/{_mixins => mixins}/hf_token.py (98%) rename src/noteflow/grpc/{_mixins => mixins}/identity.py (99%) rename src/noteflow/grpc/{_mixins => mixins}/meeting/__init__.py (100%) rename src/noteflow/grpc/{_mixins => mixins}/meeting/_post_processing.py (98%) rename src/noteflow/grpc/{_mixins => mixins}/meeting/_project_scope.py (96%) rename src/noteflow/grpc/{_mixins => mixins}/meeting/_stop_ops.py (97%) rename src/noteflow/grpc/{_mixins => mixins}/meeting/meeting_mixin.py (99%) rename src/noteflow/grpc/{_mixins => mixins}/observability.py (100%) rename src/noteflow/grpc/{_mixins => mixins}/oidc/__init__.py (100%) rename src/noteflow/grpc/{_mixins => mixins}/oidc/_support.py (100%) rename src/noteflow/grpc/{_mixins => mixins}/oidc/oidc_mixin.py (100%) rename src/noteflow/grpc/{_mixins => mixins}/preferences.py (100%) rename src/noteflow/grpc/{_mixins => mixins}/project/__init__.py (100%) rename src/noteflow/grpc/{_mixins => mixins}/project/_converters.py (100%) rename src/noteflow/grpc/{_mixins => mixins}/project/_membership.py (100%) rename src/noteflow/grpc/{_mixins => mixins}/project/_mixin.py (100%) rename src/noteflow/grpc/{_mixins => mixins}/protocols.py (100%) rename src/noteflow/grpc/{_mixins => mixins}/servicer_core/__init__.py (100%) rename src/noteflow/grpc/{_mixins => mixins}/servicer_core/protocols.py (100%) rename src/noteflow/grpc/{_mixins => mixins}/servicer_other/__init__.py (100%) rename src/noteflow/grpc/{_mixins => mixins}/servicer_other/protocols.py (98%) rename src/noteflow/grpc/{_mixins => mixins}/streaming/__init__.py (100%) rename src/noteflow/grpc/{_mixins => mixins}/streaming/_asr.py (100%) rename src/noteflow/grpc/{_mixins => mixins}/streaming/_cleanup.py (100%) rename src/noteflow/grpc/{_mixins => mixins}/streaming/_mixin.py (99%) rename src/noteflow/grpc/{_mixins => mixins}/streaming/_partials.py (100%) rename src/noteflow/grpc/{_mixins => mixins}/streaming/_processing/__init__.py (100%) rename src/noteflow/grpc/{_mixins => mixins}/streaming/_processing/_audio_ops.py (100%) rename src/noteflow/grpc/{_mixins => mixins}/streaming/_processing/_chunk_tracking.py (100%) rename src/noteflow/grpc/{_mixins => mixins}/streaming/_processing/_congestion.py (96%) rename src/noteflow/grpc/{_mixins => mixins}/streaming/_processing/_constants.py (100%) rename src/noteflow/grpc/{_mixins => mixins}/streaming/_processing/_types.py (100%) rename src/noteflow/grpc/{_mixins => mixins}/streaming/_processing/_vad_processing.py (100%) rename src/noteflow/grpc/{_mixins => mixins}/streaming/_session.py (98%) rename src/noteflow/grpc/{_mixins => mixins}/streaming/_session_helpers.py (100%) rename src/noteflow/grpc/{_mixins => mixins}/streaming/_types.py (100%) rename src/noteflow/grpc/{_mixins => mixins}/streaming_config.py (95%) rename src/noteflow/grpc/{_mixins => mixins}/summarization/__init__.py (100%) rename src/noteflow/grpc/{_mixins => mixins}/summarization/_consent.py (100%) rename src/noteflow/grpc/{_mixins => mixins}/summarization/_consent_mixin.py (100%) rename src/noteflow/grpc/{_mixins => mixins}/summarization/_constants.py (100%) rename src/noteflow/grpc/{_mixins => mixins}/summarization/_context_builders.py (100%) rename src/noteflow/grpc/{_mixins => mixins}/summarization/_generation_mixin.py (98%) rename src/noteflow/grpc/{_mixins => mixins}/summarization/_summary_generation.py (100%) rename src/noteflow/grpc/{_mixins => mixins}/summarization/_template_crud.py (100%) rename src/noteflow/grpc/{_mixins => mixins}/summarization/_template_resolution.py (100%) rename src/noteflow/grpc/{_mixins => mixins}/summarization/_templates_mixin.py (100%) rename src/noteflow/grpc/{_mixins => mixins}/sync.py (100%) rename src/noteflow/grpc/{_mixins => mixins}/webhooks.py (100%) create mode 100644 src/noteflow/grpc/server/health.py create mode 100644 src/noteflow/grpc/server/internal/__init__.py rename src/noteflow/grpc/server/{_bootstrap.py => internal/bootstrap.py} (92%) rename src/noteflow/grpc/server/{_lifecycle.py => internal/lifecycle.py} (98%) rename src/noteflow/grpc/server/{_services.py => internal/services.py} (90%) rename src/noteflow/grpc/server/{_setup.py => internal/setup.py} (86%) rename src/noteflow/grpc/server/{_streaming_config.py => internal/streaming_config.py} (96%) rename src/noteflow/grpc/server/{_types.py => internal/types.py} (80%) create mode 100644 src/noteflow/grpc/servicer/__init__.py rename src/noteflow/grpc/{_service_base.py => servicer/base.py} (92%) rename src/noteflow/grpc/{_service_mixins.py => servicer/mixins.py} (98%) rename src/noteflow/grpc/{_service_shutdown.py => servicer/shutdown.py} (98%) rename src/noteflow/grpc/{_service_stubs.py => servicer/stubs.py} (98%) create mode 100644 src/noteflow/grpc/startup/__init__.py rename src/noteflow/grpc/{_startup_banner.py => startup/banner.py} (96%) rename src/noteflow/grpc/{_server_bootstrap.py => startup/server_bootstrap.py} (100%) rename src/noteflow/grpc/{_startup_services.py => startup/services.py} (97%) rename src/noteflow/grpc/{_startup.py => startup/startup.py} (98%) create mode 100644 src/noteflow/grpc/types/__init__.py create mode 100644 src/noteflow/infrastructure/metrics/infrastructure_metrics.py create mode 100644 tests/grpc/test_client_result.py create mode 100644 tests/grpc/test_server_health.py create mode 100644 tests/grpc/test_streaming_metrics.py create mode 100644 tests/grpc/test_task_callbacks.py create mode 100644 tests/infrastructure/metrics/test_infrastructure_metrics.py create mode 100644 tests/quality/_detectors/__init__.py create mode 100644 tests/quality/_detectors/code_smells.py create mode 100644 tests/quality/_detectors/stale_code.py rename tests/quality/{_test_smell_collectors.py => _detectors/test_smells.py} (76%) create mode 100644 tests/quality/_detectors/wrappers.py diff --git a/.claude/hookify.block-broad-exception-handler.local.md b/.claude/hookify.block-broad-exception-handler.local.md new file mode 100644 index 0000000..b8d400a --- /dev/null +++ b/.claude/hookify.block-broad-exception-handler.local.md @@ -0,0 +1,61 @@ +--- +name: block-broad-exception-handler +enabled: true +event: file +conditions: + - field: new_text + operator: regex_match + pattern: except\s+Exception\s*(?:as\s+\w+)?:\s*\n\s+(?:logger\.|logging\.) +action: block +--- + +**BLOCKED: Broad `except Exception:` handler detected** + +Catching generic `Exception` and only logging creates silent failures that are nearly impossible to debug. + +**Why this is dangerous:** +- Catches ALL exceptions including programming errors (`TypeError`, `AttributeError`) +- Hides bugs that should crash loudly and be fixed immediately +- Makes the system appear to work while silently failing +- Log messages get lost in noise; exceptions should bubble up + +**Acceptable uses (require explicit justification):** +1. Top-level handlers in background tasks that MUST NOT crash +2. Fire-and-forget operations where failure is truly acceptable +3. User-provided callbacks that could raise anything + +**If you believe this is acceptable, add a comment:** +```python +# INTENTIONAL BROAD HANDLER: +# - +# - +``` + +**What to do instead:** +1. Catch specific exception types you can handle +2. Let unexpected exceptions propagate +3. Use domain-specific exceptions from `src/noteflow/domain/errors.py` + +**Resolution pattern:** +```python +# BAD: Catches everything silently +except Exception: + logger.exception("Something failed") + +# GOOD: Specific exceptions +except (ValueError, KeyError) as e: + logger.warning("Config parse error: %s", e) + raise ConfigError(str(e)) from e + +# GOOD: Let others propagate +except ValidationError as e: + logger.error("Validation failed: %s", e) + raise # Re-raise or wrap in domain error +``` + +**If truly fire-and-forget, surface metrics:** +```python +except Exception: + logger.exception("Background task failed") + metrics.increment("background_task_failures") # Observable! +``` diff --git a/.claude/hookify.block-datetime-now-fallback.local.md b/.claude/hookify.block-datetime-now-fallback.local.md new file mode 100644 index 0000000..e57ca09 --- /dev/null +++ b/.claude/hookify.block-datetime-now-fallback.local.md @@ -0,0 +1,42 @@ +--- +name: block-datetime-now-fallback +enabled: true +event: file +pattern: return\s+datetime\.now\s*\( +action: block +--- + +**BLOCKED: datetime.now() fallback detected** + +Returning `datetime.now()` as a fallback on parse failures causes **data corruption**. + +**Why this is dangerous:** +- Silent data corruption - timestamps become incorrect without any error signal +- Debugging nightmare - no way to trace back to the original parse failure +- Data integrity loss - downstream consumers receive fabricated timestamps + +**What to do instead:** +1. Return `None` and let callers handle missing timestamps explicitly +2. Raise a typed error (e.g., `DateParseError`) so failures are visible +3. Use `Result[datetime, ParseError]` pattern for explicit error handling + +**Examples from swallowed.md:** +- `_parse_google_datetime` returns `datetime.now(UTC)` on `ValueError` +- `parse_outlook_datetime` returns `datetime.now(UTC)` on `ValueError` + +**Resolution pattern:** +```python +# BAD: Silent data corruption +except ValueError: + logger.warning("Failed to parse: %s", dt_str) + return datetime.now(UTC) # Data corruption! + +# GOOD: Explicit failure +except ValueError as e: + raise DateParseError(f"Invalid datetime: {dt_str}") from e + +# GOOD: Optional return +except ValueError: + logger.warning("Failed to parse: %s", dt_str) + return None # Caller handles missing timestamp +``` diff --git a/.claude/hookify.block-default-value-swallow.local.md b/.claude/hookify.block-default-value-swallow.local.md new file mode 100644 index 0000000..5d1b2b6 --- /dev/null +++ b/.claude/hookify.block-default-value-swallow.local.md @@ -0,0 +1,60 @@ +--- +name: block-default-value-swallow +enabled: true +event: file +conditions: + - field: new_text + operator: regex_match + pattern: except\s+\w*(?:Error|Exception).*?:\s*\n\s+.*?(?:logger\.|logging\.).*?(?:warning|warn).*?\n\s+return\s+(?:\w+Settings|Defaults?\(|default_|\{[^}]*\}|[A-Z_]+_DEFAULT) +action: block +--- + +**BLOCKED: Default value swallowing on config/settings failure** + +Returning hardcoded defaults when configuration loading fails hides critical initialization errors. + +**Why this is dangerous:** +- Application runs with unexpected/incorrect configuration +- Users have no idea their settings aren't being applied +- Subtle bugs that only manifest under specific conditions +- Security settings might be weaker than intended + +**Examples from swallowed.md:** +- `get_llm_settings` returns hardcoded defaults on any exception +- `_get_ollama_settings` returns defaults on settings load failure +- `get_webhook_settings` returns defaults on exception +- `diarization_job_ttl_seconds` returns default TTL on failure + +**What to do instead:** +1. **Fail fast at startup** - validate config before accepting requests +2. **Return typed errors** - let callers decide how to handle missing config +3. **Mark degraded mode** - if defaulting is acceptable, make it visible + +**Resolution pattern:** +```python +# BAD: Silent defaulting +except Exception as exc: + logger.warning("Settings load failed, using defaults") + return HardcodedDefaults() # User has no idea! + +# GOOD: Fail fast +except Exception as exc: + raise ConfigurationError(f"Failed to load settings: {exc}") from exc + +# GOOD: Explicit degraded mode (if truly acceptable) +except Exception as exc: + logger.error("config_degraded_mode", error=str(exc)) + metrics.set_gauge("config_degraded", 1) # Observable! + return DefaultSettings(degraded=True) # Callers can check +``` + +**Startup validation pattern:** +```python +# Validate config at startup, not on every call +def validate_config_or_fail(): + try: + settings = load_settings() + settings.validate() + except Exception as e: + sys.exit(f"Configuration error: {e}") +``` diff --git a/.claude/hookify.block-silent-none-return.local.md b/.claude/hookify.block-silent-none-return.local.md new file mode 100644 index 0000000..244c561 --- /dev/null +++ b/.claude/hookify.block-silent-none-return.local.md @@ -0,0 +1,51 @@ +--- +name: block-silent-none-return +enabled: true +event: file +conditions: + - field: new_text + operator: regex_match + pattern: except\s+\w*Error.*?:\s*\n\s+.*?(?:logger\.|logging\.).*?\n\s+return\s+(?:None|\[\]|False|\{\}|0) +action: block +--- + +**BLOCKED: Silent error swallowing with default return detected** + +Catching an exception, logging it, and returning `None`/`[]`/`False`/`{}`/`0` hides failures from callers. + +**Why this is problematic:** +- Callers cannot distinguish "no result" from "error occurred" +- Error context is lost - only appears in logs, not in call stack +- Leads to cascading failures with misleading error messages +- Makes debugging significantly harder + +**What to do instead:** +1. Re-raise the exception (possibly wrapped in a domain error) +2. Return a `Result` type that explicitly contains error information +3. Use domain-specific exceptions from `src/noteflow/domain/errors.py` + +**Examples from swallowed.md:** +- gRPC client methods catch `RpcError` and return `None` +- Auth workflows catch `KeyError`/`ValueError` and return `None` +- Token refresh catches `OAuthError` and returns `None` + +**Resolution pattern:** +```python +# BAD: Silent swallowing +except RpcError as e: + logger.error("Failed to create meeting: %s", e) + return None # Caller has no idea what failed + +# GOOD: Let caller handle +except RpcError as e: + raise MeetingCreationError(f"gRPC failed: {e}") from e + +# GOOD: Result type +except RpcError as e: + logger.error("Failed to create meeting: %s", e) + return Result.failure(MeetingError.RPC_FAILURE, str(e)) +``` + +**Check these centralized helpers:** +- `src/noteflow/domain/errors.py` - `DomainError` + `ErrorCode` +- `src/noteflow/grpc/_mixins/errors/` - gRPC error helpers diff --git a/.claude/hookify.block-any-type.local.md b/.claude/hooks/hookify.block-any-type.local.md similarity index 100% rename from .claude/hookify.block-any-type.local.md rename to .claude/hooks/hookify.block-any-type.local.md diff --git a/.claude/hookify.block-assertion-roulette.local.md b/.claude/hooks/hookify.block-assertion-roulette.local.md similarity index 100% rename from .claude/hookify.block-assertion-roulette.local.md rename to .claude/hooks/hookify.block-assertion-roulette.local.md diff --git a/.claude/hookify.block-code-quality-test-bash.local.md b/.claude/hooks/hookify.block-code-quality-test-bash.local.md similarity index 100% rename from .claude/hookify.block-code-quality-test-bash.local.md rename to .claude/hooks/hookify.block-code-quality-test-bash.local.md diff --git a/.claude/hookify.block-code-quality-test-edits.local.md b/.claude/hooks/hookify.block-code-quality-test-edits.local.md similarity index 100% rename from .claude/hookify.block-code-quality-test-edits.local.md rename to .claude/hooks/hookify.block-code-quality-test-edits.local.md diff --git a/.claude/hookify.block-code-quality-test-serena-plugin.local.md b/.claude/hooks/hookify.block-code-quality-test-serena-plugin.local.md similarity index 100% rename from .claude/hookify.block-code-quality-test-serena-plugin.local.md rename to .claude/hooks/hookify.block-code-quality-test-serena-plugin.local.md diff --git a/.claude/hookify.block-code-quality-test-serena.local.md b/.claude/hooks/hookify.block-code-quality-test-serena.local.md similarity index 100% rename from .claude/hookify.block-code-quality-test-serena.local.md rename to .claude/hooks/hookify.block-code-quality-test-serena.local.md diff --git a/.claude/hookify.block-duplicate-fixtures.local.md b/.claude/hooks/hookify.block-duplicate-fixtures.local.md similarity index 100% rename from .claude/hookify.block-duplicate-fixtures.local.md rename to .claude/hooks/hookify.block-duplicate-fixtures.local.md diff --git a/.claude/hookify.block-ignore-preexisting.local.md b/.claude/hooks/hookify.block-ignore-preexisting.local.md similarity index 100% rename from .claude/hookify.block-ignore-preexisting.local.md rename to .claude/hooks/hookify.block-ignore-preexisting.local.md diff --git a/.claude/hookify.block-linter-config-frontend-bash.local.md b/.claude/hooks/hookify.block-linter-config-frontend-bash.local.md similarity index 100% rename from .claude/hookify.block-linter-config-frontend-bash.local.md rename to .claude/hooks/hookify.block-linter-config-frontend-bash.local.md diff --git a/.claude/hookify.block-linter-config-frontend.local.md b/.claude/hooks/hookify.block-linter-config-frontend.local.md similarity index 100% rename from .claude/hookify.block-linter-config-frontend.local.md rename to .claude/hooks/hookify.block-linter-config-frontend.local.md diff --git a/.claude/hookify.block-linter-config-python-bash.local.md b/.claude/hooks/hookify.block-linter-config-python-bash.local.md similarity index 100% rename from .claude/hookify.block-linter-config-python-bash.local.md rename to .claude/hooks/hookify.block-linter-config-python-bash.local.md diff --git a/.claude/hookify.block-linter-config-python.local.md b/.claude/hooks/hookify.block-linter-config-python.local.md similarity index 100% rename from .claude/hookify.block-linter-config-python.local.md rename to .claude/hooks/hookify.block-linter-config-python.local.md diff --git a/.claude/hookify.block-magic-numbers.local.md b/.claude/hooks/hookify.block-magic-numbers.local.md similarity index 100% rename from .claude/hookify.block-magic-numbers.local.md rename to .claude/hooks/hookify.block-magic-numbers.local.md diff --git a/.claude/hookify.block-makefile-bash.local.md b/.claude/hooks/hookify.block-makefile-bash.local.md similarity index 100% rename from .claude/hookify.block-makefile-bash.local.md rename to .claude/hooks/hookify.block-makefile-bash.local.md diff --git a/.claude/hookify.block-makefile-edit.local.md b/.claude/hooks/hookify.block-makefile-edit.local.md similarity index 100% rename from .claude/hookify.block-makefile-edit.local.md rename to .claude/hooks/hookify.block-makefile-edit.local.md diff --git a/.claude/hookify.block-test-loops-conditionals.local.md b/.claude/hooks/hookify.block-test-loops-conditionals.local.md similarity index 100% rename from .claude/hookify.block-test-loops-conditionals.local.md rename to .claude/hooks/hookify.block-test-loops-conditionals.local.md diff --git a/.claude/hookify.block-tests-quality-bash.local.md b/.claude/hooks/hookify.block-tests-quality-bash.local.md similarity index 100% rename from .claude/hookify.block-tests-quality-bash.local.md rename to .claude/hooks/hookify.block-tests-quality-bash.local.md diff --git a/.claude/hookify.block-tests-quality.local.md b/.claude/hooks/hookify.block-tests-quality.local.md similarity index 100% rename from .claude/hookify.block-tests-quality.local.md rename to .claude/hooks/hookify.block-tests-quality.local.md diff --git a/.claude/hookify.block-type-ignore.local.md b/.claude/hooks/hookify.block-type-ignore.local.md similarity index 100% rename from .claude/hookify.block-type-ignore.local.md rename to .claude/hooks/hookify.block-type-ignore.local.md diff --git a/.claude/hookify.require-make-quality.local.md b/.claude/hooks/hookify.require-make-quality.local.md similarity index 100% rename from .claude/hookify.require-make-quality.local.md rename to .claude/hooks/hookify.require-make-quality.local.md diff --git a/.claude/hookify.warn-baselines-edit-bash.local.md b/.claude/hooks/hookify.warn-baselines-edit-bash.local.md similarity index 100% rename from .claude/hookify.warn-baselines-edit-bash.local.md rename to .claude/hooks/hookify.warn-baselines-edit-bash.local.md diff --git a/.claude/hookify.warn-baselines-edit.local.md b/.claude/hooks/hookify.warn-baselines-edit.local.md similarity index 100% rename from .claude/hookify.warn-baselines-edit.local.md rename to .claude/hooks/hookify.warn-baselines-edit.local.md diff --git a/.claude/hookify.warn-large-file.local.md b/.claude/hooks/hookify.warn-large-file.local.md similarity index 100% rename from .claude/hookify.warn-large-file.local.md rename to .claude/hooks/hookify.warn-large-file.local.md diff --git a/.claude/hookify.warn-new-file-search.local.md b/.claude/hooks/hookify.warn-new-file-search.local.md similarity index 100% rename from .claude/hookify.warn-new-file-search.local.md rename to .claude/hooks/hookify.warn-new-file-search.local.md diff --git a/.mcp.json b/.mcp.json new file mode 100644 index 0000000..e0dcd23 --- /dev/null +++ b/.mcp.json @@ -0,0 +1,8 @@ +{ + "mcpServers": { + "lightrag-mcp": { + "type": "sse", + "url": "http://192.168.50.185:8150/sse" + } + } +} \ No newline at end of file diff --git a/.rag/01-architecture-overview.md b/.rag/01-architecture-overview.md new file mode 100644 index 0000000..d1908d9 --- /dev/null +++ b/.rag/01-architecture-overview.md @@ -0,0 +1,69 @@ +# NoteFlow Architecture Overview + +## Project Type +Intelligent meeting notetaker: local-first audio capture + navigable recall + evidence-linked summaries. + +## Tech Stack +- **Python Backend**: gRPC server, domain logic, infrastructure adapters (src/noteflow/) +- **Tauri Desktop Client**: Rust IPC + React UI (client/) +- **Database**: PostgreSQL with pgvector extension, async SQLAlchemy + asyncpg + +## Architecture Pattern +Hexagonal (Ports & Adapters): +- **Domain Layer** (`domain/`): Pure business logic, entities, value objects, ports (protocols) +- **Application Layer** (`application/`): Use-cases/services, orchestration +- **Infrastructure Layer** (`infrastructure/`): Implementations (repos, ASR, auth, persistence) +- **gRPC Layer** (`grpc/`): API boundary, server mixins, proto definitions + +## Key Entry Points +| Entry Point | Description | +|-------------|-------------| +| `python -m noteflow.grpc.server` | Backend server | +| `cd client && npm run dev` | Web UI (Vite) | +| `cd client && npm run tauri dev` | Desktop Tauri dev | + +## Directory Structure +``` +src/noteflow/ +├── domain/ # Entities, ports, value objects +├── application/ # Use-cases/services +├── infrastructure/ # Implementations +├── grpc/ # gRPC layer +├── config/ # Settings +└── cli/ # CLI tools + +client/src/ +├── api/ # API adapters & types +├── hooks/ # Custom React hooks +├── contexts/ # React contexts +├── components/ # UI components +├── pages/ # Route pages +└── lib/ # Utilities + +client/src-tauri/src/ +├── commands/ # Tauri IPC handlers +├── grpc/ # gRPC client & types +├── state/ # Runtime state +├── audio/ # Audio capture/playback +└── crypto/ # Encryption +``` + +## Proto/gRPC Contract +Proto file: `src/noteflow/grpc/proto/noteflow.proto` +Regenerate after changes: +```bash +python -m grpc_tools.protoc -I src/noteflow/grpc/proto \ + --python_out=src/noteflow/grpc/proto \ + --grpc_python_out=src/noteflow/grpc/proto \ + src/noteflow/grpc/proto/noteflow.proto +python scripts/patch_grpc_stubs.py +``` + +## Quality Commands +```bash +make quality # ALL checks (TS + Rust + Python) +make quality-py # Python: lint + type-check + test-quality +make quality-ts # TypeScript: type-check + lint +make quality-rs # Rust: clippy + lint +pytest tests/quality/ # After any non-trivial changes +``` diff --git a/.rag/02-domain-entities.md b/.rag/02-domain-entities.md new file mode 100644 index 0000000..958c3d1 --- /dev/null +++ b/.rag/02-domain-entities.md @@ -0,0 +1,67 @@ +# NoteFlow Domain Entities + +## Location +`src/noteflow/domain/entities/` and `domain/value_objects.py` + +## Core Entities + +### Meeting (`entities/meeting.py`) +Aggregate root with lifecycle states. +- **Classes**: `MeetingLoadParams`, `MeetingCreateParams`, `Meeting` +- **States**: CREATED → RECORDING → STOPPED → COMPLETED (or ERROR, STOPPING) +- **Key Fields**: id (UUID), title, state, project_id, created_at, started_at, stopped_at + +### Segment (`entities/segment.py`) +Transcript fragment with timing and speaker. +- **Key Fields**: segment_id, text, start_time, end_time, speaker_id, language, confidence +- **Related**: `WordTiming` for word-level boundaries + +### Summary (`entities/summary.py`) +Generated summary with key points and action items. +- **Key Fields**: id, meeting_id, template_id, content, format, provider +- **Contains**: `KeyPoint[]`, `ActionItem[]` + +### Annotation (`entities/annotation.py`) +User annotation linked to segments. +- **Types**: ACTION_ITEM, DECISION, NOTE, RISK +- **Key Fields**: id, meeting_id, type, text, segment_ids, priority + +### NamedEntity (`entities/named_entity.py`) +NER extraction results. +- **Categories**: PERSON, COMPANY, PRODUCT, TECHNICAL, ACRONYM, LOCATION, DATE, OTHER +- **Key Fields**: id, name, category, segment_ids, meeting_id + +### Project (`entities/project.py`) +Workspace grouping for meetings. +- **Contains**: `ExportRules`, `TriggerRules` +- **Key Fields**: id, workspace_id, name, slug, members + +### Integration (`entities/integration.py`) +External service connections. +- **Types**: AUTH, EMAIL, CALENDAR, PKM, CUSTOM +- **Statuses**: DISCONNECTED, CONNECTED, ERROR + +### SummarizationTemplate (`entities/summarization_template.py`) +Configurable summary generation template. +- **Key Fields**: id, name, tone, format, verbosity + +## Value Objects (`value_objects.py`) + +### Type-Safe IDs +- `MeetingId = NewType("MeetingId", UUID)` +- `AnnotationId = NewType("AnnotationId", UUID)` + +### Enums +- `MeetingState` (IntEnum): UNSPECIFIED=0, CREATED=1, RECORDING=2, STOPPED=3, COMPLETED=4, ERROR=5, STOPPING=6 +- `AnnotationType` (Enum): ACTION_ITEM, DECISION, NOTE, RISK +- `ExportFormat` (Enum): MARKDOWN, HTML, PDF + +## Error Hierarchy (`errors.py`) +- Base: `DomainError` with `ErrorCode` enum mapping to gRPC `StatusCode` +- 30+ specific error types (MEETING_NOT_FOUND, WORKSPACE_ACCESS_DENIED, etc.) + +## Identity Entities (`domain/identity/`) +- `User`: User identity with email, name, picture +- `Workspace`: Tenant container +- `WorkspaceMembership`: User-workspace relationship with role +- `ProjectRole`: Role definitions (owner, editor, viewer) diff --git a/.rag/03-domain-ports.md b/.rag/03-domain-ports.md new file mode 100644 index 0000000..e60b313 --- /dev/null +++ b/.rag/03-domain-ports.md @@ -0,0 +1,133 @@ +# NoteFlow Domain Ports (Protocols) + +## Location +`src/noteflow/domain/ports/` + +## Repository Protocols (`repositories/`) + +### Core Repositories (`transcript.py`) +| Protocol | Key Methods | +|----------|-------------| +| `MeetingRepository` | `add()`, `get()`, `list()`, `update()`, `delete()`, `count_by_state()`, `find_older_than()` | +| `SegmentRepository` | `add()`, `add_batch()`, `get()`, `list_by_meeting()`, `update()`, `delete()` | +| `SummaryRepository` | `add()`, `get()`, `list_by_meeting()`, `mark_verified()` | +| `AnnotationRepository` | `add()`, `get()`, `list_by_meeting()`, `update()`, `delete()` | + +### External Repositories +| Protocol | Location | Purpose | +|----------|----------|---------| +| `AssetRepository` | `asset.py` | Store/retrieve meeting audio files | +| `DiarizationJobRepository` | `background.py` | Track background diarization jobs | +| `EntityRepository` | `external/_entity.py` | Persist NER entities | +| `IntegrationRepository` | `external/_integration.py` | Store OAuth integrations | +| `WebhookRepository` | `external/_webhook.py` | Webhook configs and delivery logs | +| `UsageEventRepository` | `external/_usage.py` | Track usage metrics | + +### Identity Repositories (`identity/`) +| Protocol | Purpose | +|----------|---------| +| `UserRepository` | User identity and authentication | +| `WorkspaceRepository` | Workspace tenancy | +| `ProjectRepository` | Project CRUD and member access | +| `ProjectMembershipRepository` | Project role-based access | +| `SummarizationTemplateRepository` | Template CRUD and versioning | + +## Engine/Provider Protocols + +### DiarizationEngine (`diarization.py`) +Speaker identification (streaming: diart, offline: pyannote) +- `assign_speakers(audio: ndarray, segments: list[Segment]) -> list[SpeakerAssignment]` +- `is_ready() -> bool` + +### NerPort (`ner.py`) +Named entity recognition with spaCy +- `extract(text: str) -> list[NamedEntity]` +- `extract_from_segments(segments: list[Segment]) -> list[NamedEntity]` +- `is_ready() -> bool` + +### OAuthPort (`calendar.py`) +OAuth PKCE flow +- `initiate_auth(provider: str) -> AuthUrl` +- `complete_auth(code: str, state: str) -> TokenResponse` + +### CalendarProvider (`calendar.py`) +Calendar event fetching +- `list_events(start: datetime, end: datetime) -> list[CalendarEventInfo]` + +### SummarizerProvider (`summarization/ports.py`) +LLM summarization +- `request(context: SummarizationRequest) -> SummarizationResult` + +## Unit of Work Pattern (`unit_of_work.py`) + +### Hierarchical Protocol Structure +```python +UnitOfWorkCapabilities +├── supports_annotations: bool +├── supports_diarization_jobs: bool +├── supports_preferences: bool +├── supports_entities: bool +├── supports_integrations: bool +├── supports_webhooks: bool +├── supports_usage_events: bool +├── supports_users: bool +├── supports_workspaces: bool +├── supports_projects: bool + +UnitOfWorkCoreRepositories +├── meetings: MeetingRepository +├── segments: SegmentRepository +├── summaries: SummaryRepository +├── assets: AssetRepository + +UnitOfWorkOptionalRepositories +├── annotations: AnnotationRepository | None +├── diarization_jobs: DiarizationJobRepository | None +├── preferences: PreferencesRepository | None +├── entities: EntityRepository | None +├── integrations: IntegrationRepository | None +├── webhooks: WebhookRepository | None +├── usage_events: UsageEventRepository | None + +UnitOfWorkLifecycle +├── __aenter__() / __aexit__() +├── commit() async +├── rollback() async +``` + +## Key Data Classes + +### SummarizationRequest +```python +@dataclass +class SummarizationRequest: + meeting_id: MeetingId + segments: list[Segment] + template: SummarizationTemplate | None + context: dict[str, str] +``` + +### SummarizationResult +```python +@dataclass +class SummarizationResult: + summary: str + key_points: list[KeyPoint] + action_items: list[ActionItem] + model_name: str + provider_name: str + tokens_used: int +``` + +### CalendarEventInfo +```python +@dataclass +class CalendarEventInfo: + id: str + title: str + start_time: datetime + end_time: datetime + attendees: list[str] + location: str | None + description: str | None +``` diff --git a/.rag/04-application-services.md b/.rag/04-application-services.md new file mode 100644 index 0000000..6d943ff --- /dev/null +++ b/.rag/04-application-services.md @@ -0,0 +1,142 @@ +# NoteFlow Application Services + +## Location +`src/noteflow/application/services/` + +## Core Services + +### MeetingService (`meeting/`) +Meeting lifecycle, segments, annotations, summaries, state. + +**Files**: +- `meeting_service.py` — Composite class combining all mixins +- `_base.py` — Core initialization and dependencies +- `_crud_mixin.py` — Create, read, update, delete operations +- `_segments_mixin.py` — Segment management +- `_summaries_mixin.py` — Summary operations +- `_state_mixin.py` — State machine transitions +- `_annotations_mixin.py` — Annotation CRUD +- `_types.py` — Service-specific TypedDicts + +**Key Methods**: +- `create_meeting(title, project_id) -> Meeting` +- `get_meeting(meeting_id) -> Meeting` +- `list_meetings(filters) -> list[Meeting]` +- `stop_meeting(meeting_id) -> Meeting` +- `add_segment(meeting_id, segment_data) -> Segment` +- `add_annotation(meeting_id, annotation_data) -> Annotation` +- `generate_summary(meeting_id, template_id) -> Summary` + +### IdentityService (`identity/`) +User/workspace context, defaults, tenancy scoping. + +**Files**: +- `identity_service.py` — Main service +- `_context_mixin.py` — Request context handling +- `_workspace_mixin.py` — Workspace operations +- `_defaults_mixin.py` — Default user/workspace creation + +**Key Methods**: +- `get_current_user() -> User` +- `get_current_workspace() -> Workspace` +- `switch_workspace(workspace_id) -> Workspace` +- `ensure_defaults() -> tuple[User, Workspace]` + +### CalendarService (`calendar/`) +OAuth integration, event fetching, sync management. + +**Files**: +- `calendar_service.py` — Main service +- `_connection_mixin.py` — OAuth connection handling +- `_events_mixin.py` — Event fetching +- `_oauth_mixin.py` — OAuth flow management +- `_service_mixin.py` — Provider configuration +- `_errors.py` — Calendar-specific errors + +**Key Methods**: +- `initiate_oauth(provider) -> AuthUrl` +- `complete_oauth(code, state) -> Integration` +- `list_events(start, end) -> list[CalendarEvent]` +- `get_connection_status() -> ConnectionStatus` + +### SummarizationService (`summarization/`) +Summary generation, template management, cloud consent. + +**Files**: +- `summarization_service.py` — Main service +- `template_service.py` — Template CRUD +- `consent_manager.py` — Cloud consent flow + +**Key Methods**: +- `generate_summary(meeting_id, template_id) -> Summary` +- `create_template(data) -> SummarizationTemplate` +- `list_templates() -> list[SummarizationTemplate]` +- `grant_consent() -> bool` +- `revoke_consent() -> bool` +- `has_consent() -> bool` + +### ProjectService (`project_service/`) +Project CRUD, member management, roles, rules. + +**Files**: +- `__init__.py` — Main service export +- `crud.py` — Project CRUD operations +- `members.py` — Member management +- `roles.py` — Role-based access +- `rules.py` — Project rules configuration +- `active.py` — Active project tracking +- `_types.py` — Service types + +**Key Methods**: +- `create_project(name, workspace_id) -> Project` +- `add_member(project_id, user_id, role) -> ProjectMembership` +- `update_member_role(project_id, user_id, role) -> ProjectMembership` +- `remove_member(project_id, user_id) -> bool` +- `set_active_project(project_id) -> Project` + +## Supporting Services + +### NerService (`ner_service.py`) +Named entity extraction wrapper, model loading. +- `extract_entities(meeting_id) -> list[NamedEntity]` +- `is_ready() -> bool` + +### ExportService (`export_service.py`) +Transcript export (Markdown, HTML, PDF). +- `export(meeting_id, format) -> ExportResult` + +### WebhookService (`webhook_service.py`) +Webhook registration, delivery, retry logic. +- `register_webhook(config) -> WebhookConfig` +- `deliver_event(event) -> WebhookDelivery` + +### AuthService (`auth_service.py`) +User authentication, OIDC integration. +- `initiate_login(provider) -> AuthUrl` +- `complete_login(code, state) -> User` +- `logout() -> bool` + +### TriggerService (`trigger_service.py`) +Calendar/audio/foreground-app trigger detection. +- `check_triggers() -> list[TriggerSignal]` +- `snooze_triggers(duration) -> bool` + +### RetentionService (`retention_service.py`) +Automatic meeting deletion based on policy. +- `apply_retention_policy() -> int` + +### RecoveryService (`recovery/`) +Data recovery (meeting, job, audio). +- `recover_meeting(meeting_id) -> Meeting` +- `recover_job(job_id) -> DiarizationJob` + +### AsrConfigService (`asr_config_service.py`) +ASR model configuration and state. +- `get_config() -> AsrConfig` +- `update_config(config) -> AsrConfig` + +### HfTokenService (`hf_token_service.py`) +Hugging Face token management. +- `set_token(token) -> bool` +- `get_status() -> HfTokenStatus` +- `validate_token() -> bool` diff --git a/.rag/05-infrastructure-adapters.md b/.rag/05-infrastructure-adapters.md new file mode 100644 index 0000000..730f99e --- /dev/null +++ b/.rag/05-infrastructure-adapters.md @@ -0,0 +1,153 @@ +# NoteFlow Infrastructure Adapters + +## Location +`src/noteflow/infrastructure/` + +## Audio & ASR Layer (`asr/`, `audio/`) + +### ASR Engine (`asr/engine.py`) +Faster-whisper wrapper for streaming ASR. +- `transcribe(audio: ndarray) -> TranscriptionResult` +- `is_ready() -> bool` + +### ASR Streaming VAD (`asr/streaming_vad.py`) +Voice activity detection for streaming. +- `process_audio(chunk: ndarray) -> list[VadSegment]` + +### ASR Segmenter (`asr/segmenter/`) +Audio segmentation into speech chunks. + +### Audio Capture (`audio/`) +Sounddevice capture, ring buffer, VU levels, playback, writer, reader. + +## Diarization Layer (`diarization/`) + +### Session Management (`session.py`) +Speaker session lifecycle with audio buffering. +- `process_audio(chunk) -> list[SpeakerTurn]` + +### Diarization Engine (`engine/`) +- **Streaming**: diart for real-time speaker detection +- **Offline**: pyannote.audio for post-meeting refinement + +### Speaker Assigner (`assigner.py`) +Assign speech segments to speaker IDs. + +## Summarization Layer (`summarization/`) + +### Cloud Provider (`cloud_provider/`) +Anthropic/OpenAI API integration. +- `generate(request) -> SummarizationResult` + +### Ollama Provider (`ollama_provider.py`) +Local Ollama LLM. +- `generate(request) -> SummarizationResult` + +### Mock Provider (`mock_provider.py`) +Testing provider. + +### Citation Verifier (`citation_verifier.py`) +Validate summary citations against transcript. +- `verify(summary, segments) -> CitationVerificationResult` + +### Template Renderer (`template_renderer.py`) +Render summary templates. + +## NER Engine (`ner/`) +spaCy-based named entity extraction. +- `extract(text) -> list[Entity]` +- `extract_from_segments(segments) -> list[NamedEntity]` + +## Persistence Layer (`persistence/`) + +### Database (`database.py`) +Async SQLAlchemy engine, session factory, pgvector support. +- `create_async_engine(url) -> AsyncEngine` +- `create_async_session_factory(engine) -> async_sessionmaker` + +### ORM Models (`models/`) +``` +core/ # MeetingModel, SegmentModel, SummaryModel, AnnotationModel +identity/ # UserModel, WorkspaceModel, ProjectModel, MembershipModel +entities/ # NamedEntityModel, SpeakerModel +integrations/ # IntegrationModel, CalendarEventModel, WebhookConfigModel +organization/ # SummarizationTemplateModel, TaskModel, TagModel +observability/ # UsageEventModel +``` + +### Base Repository (`repositories/_base/`) +```python +class BaseRepository: + async def _execute_scalar(query) -> T | None + async def _execute_scalars(query) -> list[T] + async def _add_and_flush(entity) -> T +``` + +### Unit of Work (`unit_of_work/`) +Transaction management, repository coordination. + +## Auth Layer (`auth/`) + +### OIDC Discovery (`oidc_discovery.py`) +Discover provider endpoints. +- `discover(issuer_url) -> OidcConfig` + +### OIDC Registry (`oidc_registry.py`) +Manage configured OIDC providers. +- `register(provider) -> OidcProvider` +- `list_providers() -> list[OidcProvider]` + +### OIDC Presets (`_presets.py`) +Pre-configured providers (Google, Outlook, etc.). + +## Export Layer (`export/`) + +### Markdown Export (`markdown.py`) +Convert transcript to Markdown. + +### HTML Export (`html.py`) +Convert transcript to HTML. + +### PDF Export (`pdf.py`) +Convert transcript to PDF (WeasyPrint). + +## Converters (`converters/`) +Bidirectional conversion between layers: +- ORM ↔ Domain entities +- ASR engine output ↔ Domain entities +- Calendar API ↔ Domain entities +- Webhook payloads ↔ Domain entities +- NER output ↔ Domain entities + +## Calendar Integration (`calendar/`) +Google/Outlook OAuth adapters with event API integration. + +## Security (`security/`) + +### Keystore (`keystore.py`) +AES-GCM encryption with keyring backend. +- `encrypt(data) -> bytes` +- `decrypt(data) -> bytes` + +### Crypto Utilities (`crypto/`) +Cryptographic helpers. + +## Logging & Observability + +### Log Buffer (`logging/log_buffer.py`) +In-memory log buffering for client retrieval. + +### OpenTelemetry (`observability/otel.py`) +Distributed tracing. + +### Usage Events (`observability/`) +Track usage metrics. + +### Metrics (`metrics/`) +Metric collection utilities. + +## Webhooks (`webhooks/`) + +### WebhookExecutor +Delivery, signing (HMAC-SHA256), retry with exponential backoff. +- `deliver(config, payload) -> WebhookDelivery` diff --git a/.rag/06-grpc-layer.md b/.rag/06-grpc-layer.md new file mode 100644 index 0000000..e24166d --- /dev/null +++ b/.rag/06-grpc-layer.md @@ -0,0 +1,140 @@ +# NoteFlow gRPC Layer + +## Location +`src/noteflow/grpc/` + +## Core Files + +### Service (`service.py`) +Main `NoteFlowServicer` - gRPC service implementation composing all mixins. + +### Client (`client.py`) +Python gRPC client wrapper for testing and internal use. + +### Proto (`proto/noteflow.proto`) +Service definition with bidirectional streaming and RPC methods. + +**Key RPC Groups**: +- **Streaming**: `StreamTranscription(AudioChunk) → TranscriptUpdate` (bidirectional) +- **Meeting Lifecycle**: CreateMeeting, StopMeeting, ListMeetings, GetMeeting, DeleteMeeting +- **Summaries**: GenerateSummary, ListSummarizationTemplates, CreateSummarizationTemplate +- **Diarization**: RefineSpeakerDiarization, RenameSpeaker, GetDiarizationJobStatus +- **Annotations**: AddAnnotation, GetAnnotation, ListAnnotations, UpdateAnnotation, DeleteAnnotation +- **Export**: ExportTranscript (Markdown, HTML, PDF) +- **Calendar**: Calendar event sync and OAuth +- **Webhooks**: Webhook config and delivery management +- **OIDC**: Authentication via OpenID Connect + +## Server Mixins (`_mixins/`) + +### Streaming Mixin (`streaming/`) +Bidirectional ASR streaming. + +**Files**: +- `_mixin.py` — Main StreamingMixin +- `_session.py` — Stream session lifecycle +- `_asr.py` — ASR engine integration +- `_processing/` — Audio processing pipeline (VAD, chunk tracking, congestion) +- `_partials.py` — Partial transcript handling +- `_cleanup.py` — Resource cleanup + +### Diarization Mixin (`diarization/`) +Speaker diarization (streaming + offline refinement). + +**Files**: +- `_mixin.py` — Main DiarizationMixin +- `_jobs.py` — Background job management +- `_streaming.py` — Real-time diarization +- `_refinement.py` — Offline refinement with pyannote +- `_speaker.py` — Speaker assignment +- `_status.py` — Job status tracking + +### Summarization Mixin (`summarization/`) +Summary generation and templates. + +**Files**: +- `_generation_mixin.py` — Summary generation flow +- `_templates_mixin.py` — Template CRUD +- `_consent_mixin.py` — Cloud consent handling +- `_summary_generation.py` — Core generation logic +- `_template_resolution.py` — Template lookup +- `_context_builders.py` — Context preparation + +### Meeting Mixin (`meeting/`) +Meeting lifecycle management. + +**Files**: +- `meeting_mixin.py` — Meeting state management +- `_project_scope.py` — Project scoping +- `_stop_ops.py` — Stop operations + +### Other Mixins +- `project/` — Project management +- `oidc/` — OpenID Connect auth +- `identity/` — User/workspace identity +- `annotation.py` — Segment annotations CRUD +- `export.py` — Export operations +- `entities.py` — Named entity extraction +- `calendar.py` — Calendar sync +- `webhooks.py` — Webhook management +- `preferences.py` — User preferences +- `observability.py` — Usage tracking, metrics +- `sync.py` — State synchronization + +### Error Helpers (`errors/`) +- `_abort.py` — `abort_not_found()`, `abort_invalid_argument()` +- `_require.py` — Precondition checks +- `_fetch.py` — Fetch with error handling +- `_parse.py` — Parsing helpers + +### Converters (`converters/`) +Proto ↔ Domain conversion. +- `_domain.py` — Domain entity conversion +- `_timestamps.py` — Timestamp conversion +- `_id_parsing.py` — ID parsing and validation +- `_external.py` — External entity conversion +- `_oidc.py` — OIDC entity conversion + +## Server Bootstrap + +### Files +- `_server_bootstrap.py` — gRPC server creation +- `_startup.py` — Server startup sequence +- `_startup_services.py` — Service initialization +- `_startup_banner.py` — Startup logging +- `_service_shutdown.py` — Graceful shutdown +- `_service_stubs.py` — gRPC stub management + +### Interceptors (`interceptors/`) +gRPC interceptors for identity context propagation. + +## Client Mixins (`_client_mixins/`) +Client-side gRPC operations. + +- `streaming.py` — Client streaming operations +- `meeting.py` — Meeting CRUD operations +- `diarization.py` — Diarization requests +- `export.py` — Export requests +- `annotation.py` — Annotation operations +- `converters.py` — Response converters +- `protocols.py` — ClientHost protocol + +## Critical Paths + +### Recording Flow +1. Client: `StreamTranscription(AudioChunk)` → gRPC streaming +2. Server: StreamingMixin consumes chunks +3. ASR Engine: Transcribe via faster-whisper +4. VAD: Segment by silence +5. Diarization: Assign speakers (streaming) +6. Repository: Persist Segments +7. Client: Receive `TranscriptUpdate` with segments + +### Summary Generation Flow +1. Client: `GenerateSummary(meeting_id)` → gRPC call +2. SummarizationService: Fetch segments +3. SummarizerProvider: Call LLM +4. Citation Verifier: Validate claims +5. Template Renderer: Apply template +6. Repository: Persist Summary +7. Client: Receive Summary diff --git a/.rag/07-typescript-api-layer.md b/.rag/07-typescript-api-layer.md new file mode 100644 index 0000000..fefc408 --- /dev/null +++ b/.rag/07-typescript-api-layer.md @@ -0,0 +1,172 @@ +# NoteFlow TypeScript API Layer + +## Location +`client/src/api/` + +## Architecture +Multi-adapter design with fallback chain: +1. **TauriAdapter** (`tauri-adapter.ts`) — Primary: Rust IPC to gRPC +2. **CachedAdapter** (`cached-adapter.ts`) — Fallback: Read-only cache +3. **MockAdapter** (`mock-adapter.ts`) — Development: Simulated responses + +## API Interface (`interface.ts`) + +```typescript +interface NoteFlowAPI { + // Connection + connect(): Promise; + disconnect(): Promise; + isConnected(): boolean; + getEffectiveServerUrl(): Promise; + + // Meetings + createMeeting(request: CreateMeetingRequest): Promise; + getMeeting(request: GetMeetingRequest): Promise; + listMeetings(request: ListMeetingsRequest): Promise; + stopMeeting(request: StopMeetingRequest): Promise; + deleteMeeting(request: DeleteMeetingRequest): Promise; + + // Streaming + startTranscription(meetingId: string): TranscriptionStream; + + // Diarization + refineSpeakerDiarization(request: RefineDiarizationRequest): Promise; + getDiarizationJobStatus(request: GetJobStatusRequest): Promise; + renameSpeaker(request: RenameSpeakerRequest): Promise; + + // Summaries + generateSummary(request: GenerateSummaryRequest): Promise; + listSummarizationTemplates(): Promise; + createSummarizationTemplate(request: CreateTemplateRequest): Promise; + + // Annotations + addAnnotation(request: AddAnnotationRequest): Promise; + listAnnotations(request: ListAnnotationsRequest): Promise; + updateAnnotation(request: UpdateAnnotationRequest): Promise; + deleteAnnotation(request: DeleteAnnotationRequest): Promise; + + // Export + exportTranscript(request: ExportRequest): Promise; + + // ... 50+ more methods +} +``` + +## Transcription Streaming + +```typescript +interface TranscriptionStream { + send(chunk: AudioChunk): void; + onUpdate(callback: (update: TranscriptUpdate) => void): Promise | void; + onError?(callback: (error: StreamError) => void): void; + onCongestion?(callback: (state: CongestionState) => void): void; + close(): void; +} +``` + +## Connection State (`connection-state.ts`) + +```typescript +type ConnectionMode = 'connected' | 'disconnected' | 'cached' | 'mock' | 'reconnecting'; + +interface ConnectionState { + mode: ConnectionMode; + lastConnectedAt: Date | null; + disconnectedAt: Date | null; + reconnectAttempts: number; + error: string | null; + serverUrl: string | null; +} +``` + +## Type Definitions (`types/`) + +### Core Types (`core.ts`) +- `Meeting`, `FinalSegment`, `WordTiming`, `Summary`, `Annotation` +- `KeyPoint`, `ActionItem`, `Speaker` + +### Enums (`enums.ts`) +- `UpdateType`: partial | final | vad_start | vad_end +- `MeetingState`: created | recording | stopped | completed | error +- `JobStatus`: queued | running | completed | failed | cancelled +- `AnnotationType`: action_item | decision | note | risk +- `ExportFormat`: markdown | html | pdf + +### Feature Types (`features/`) +- `webhooks.ts` — WebhookConfig, WebhookDelivery +- `calendar.ts` — CalendarProvider, CalendarEvent, OAuthConfig +- `ner.ts` — Entity extraction types +- `identity.ts` — User, Workspace +- `oidc.ts` — OIDCProvider, OIDCConfig +- `sync.ts` — SyncStatus, SyncHistory +- `observability.ts` — LogEntry, MetricPoint + +### Projects (`projects.ts`) +- `Project`, `ProjectMember`, `ProjectMembership` + +## Cached Adapter (`cached/`) + +Provides offline read-only access: + +```typescript +// cached/readonly.ts +export function rejectReadOnly(): never { + throw new Error('This action requires an active connection'); +} + +// Pattern in cached adapters +export const cachedMeetings = { + async getMeeting(id: string): Promise { + return meetingCache.get(id) ?? rejectReadOnly(); + }, + async createMeeting(): Promise { + return rejectReadOnly(); // Mutations blocked + }, +}; +``` + +### Cache Modules +- `meetings.ts` — Meeting cache with TTL +- `projects.ts` — Project cache +- `diarization.ts` — Job status cache +- `annotations.ts` — Annotation cache +- `templates.ts` — Template cache +- `preferences.ts` — Preferences cache + +## API Initialization + +```typescript +// api/index.ts +export async function initializeAPI(): Promise { + try { + const tauriAdapter = await createTauriAdapter(); + return tauriAdapter; + } catch { + console.warn('Falling back to mock adapter'); + return createMockAdapter(); + } +} + +export function getAPI(): NoteFlowAPI { + return window.__NOTEFLOW_API__ ?? mockAdapter; +} +``` + +## Usage Pattern + +```typescript +import { getAPI } from '@/api'; + +const api = getAPI(); +const meeting = await api.createMeeting({ title: 'Sprint Planning' }); +const stream = api.startTranscription(meeting.id); + +stream.onUpdate((update) => { + if (update.update_type === 'final') { + console.log('New segment:', update.segment); + } +}); + +// Send audio chunks +stream.send({ audio_data: audioBuffer }); +``` diff --git a/.rag/08-typescript-hooks-contexts.md b/.rag/08-typescript-hooks-contexts.md new file mode 100644 index 0000000..5a18766 --- /dev/null +++ b/.rag/08-typescript-hooks-contexts.md @@ -0,0 +1,212 @@ +# NoteFlow TypeScript Hooks & Contexts + +## Location +`client/src/hooks/` and `client/src/contexts/` + +## React Contexts + +### Connection Context (`connection-context.tsx`) +gRPC connection state and mode detection. + +```typescript +interface ConnectionHelpers { + state: ConnectionState; + mode: ConnectionMode; // connected | disconnected | cached | mock | reconnecting + isConnected: boolean; + isReadOnly: boolean; // cached | disconnected | mock | reconnecting + isReconnecting: boolean; + isSimulating: boolean; // Simulation mode from preferences +} + +// Usage +const { isConnected, isReadOnly, mode } = useConnection(); +``` + +### Workspace Context (`workspace-context.tsx`) +User and workspace state. + +```typescript +interface WorkspaceContextValue { + currentWorkspace: Workspace | null; + workspaces: Workspace[]; + currentUser: GetCurrentUserResponse | null; + switchWorkspace: (workspaceId: string) => Promise; + isLoading: boolean; + error: string | null; +} + +// Usage +const { currentWorkspace, currentUser, switchWorkspace } = useWorkspace(); +``` + +### Project Context (`project-context.tsx`) +Active project and project list. + +```typescript +interface ProjectContextValue { + projects: Project[]; + activeProject: Project | null; + switchProject: (projectId: string) => Promise; + isLoading: boolean; + error: string | null; +} + +// Usage +const { activeProject, projects, switchProject } = useProjects(); +``` + +## Custom Hooks + +### Diarization (`use-diarization.ts`) +Diarization job lifecycle with polling and recovery. + +```typescript +interface UseDiarizationOptions { + onComplete?: (status: DiarizationJobStatus) => void; + onError?: (error: string) => void; + pollInterval?: number; + maxRetries?: number; + showToasts?: boolean; + autoRecover?: boolean; // Auto-recovery after restart +} + +interface DiarizationState { + jobId: string | null; + status: JobStatus | null; + progress: number; // 0-100 + error: string | null; + speakerIds: string[]; + segmentsUpdated: number; + isActive: boolean; +} + +function useDiarization(options?: UseDiarizationOptions): { + state: DiarizationState; + start: (meetingId: string, numSpeakers?: number) => Promise; + cancel: () => Promise; + reset: () => void; + recover: () => Promise; +} +``` + +### Audio Devices (`use-audio-devices.ts`) +Audio device enumeration and selection. + +```typescript +interface AudioDevice { + id: string; + name: string; + kind: 'input' | 'output'; +} + +function useAudioDevices(options: UseAudioDevicesOptions): { + devices: AudioDevice[]; + selectedInput: AudioDevice | null; + selectedOutput: AudioDevice | null; + setSelectedInput: (id: string) => void; + setSelectedOutput: (id: string) => void; + isLoading: boolean; +} +``` + +### Project Hooks +- `useProject()` — Access project from context +- `useActiveProject()` — Get active project +- `useProjectMembers()` — Project membership queries + +### Cloud Consent (`use-cloud-consent.ts`) +Cloud AI consent state management. + +```typescript +function useCloudConsent(): { + hasConsent: boolean; + grantConsent: () => Promise; + revokeConsent: () => Promise; + isLoading: boolean; +} +``` + +### Integration Hooks +- `useWebhooks()` — Webhook CRUD +- `useEntityExtraction()` — NER extraction & updates +- `useCalendarSync()` — Calendar integration sync +- `useOAuthFlow()` — OAuth authentication flow +- `useAuthFlow()` — General auth flow +- `useOidcProviders()` — OIDC provider management +- `useIntegrationSync()` — Integration sync state polling +- `useIntegrationValidation()` — Integration config validation + +### Recording Hooks +- `useRecordingAppPolicy()` — App recording policy detection +- `usePostProcessing()` — Post-recording processing state + +### Utility Hooks +- `useAsyncData()` — Generic async data loading with retry +- `useGuardedMutation()` — Mutation with offline/permissions guard +- `useToast()` — Toast notifications (shadcn/ui) +- `usePanelPreferences()` — Panel layout preferences +- `useMobile()` — Mobile/responsive detection + +## Hook Patterns + +### Polling with Backoff (Diarization) +```typescript +const { state, start, cancel, recover } = useDiarization({ + pollInterval: 2000, + maxRetries: 10, + autoRecover: true, + onComplete: (status) => { + toast.success(`Diarization complete: ${status.segmentsUpdated} segments updated`); + }, +}); + +// Start job +await start(meetingId, 2); // 2 speakers + +// Monitor progress +useEffect(() => { + console.log(`Progress: ${state.progress}%`); +}, [state.progress]); +``` + +### Async Data Loading +```typescript +const { data, isLoading, error, retry } = useAsyncData( + () => getAPI().getMeeting({ meeting_id: meetingId }), + { + onError: (e) => toast.error(e.message), + deps: [meetingId], + } +); +``` + +### Connection-Aware Components +```typescript +function MyComponent() { + const { isConnected, isReadOnly } = useConnection(); + const { activeProject } = useProjects(); + + if (isReadOnly) { + return ; + } + + return ; +} +``` + +## Context Provider Pattern + +```typescript +// Root app setup +function App() { + return ( + + + + + + + + ); +} +``` diff --git a/.rag/09-typescript-components-pages.md b/.rag/09-typescript-components-pages.md new file mode 100644 index 0000000..f2bd894 --- /dev/null +++ b/.rag/09-typescript-components-pages.md @@ -0,0 +1,129 @@ +# NoteFlow TypeScript Components & Pages + +## Location +`client/src/components/` and `client/src/pages/` + +## Component Architecture + +### UI Components (`components/ui/`) +30+ shadcn/ui primitives: button, input, dialog, form, toast, etc. + +### Recording Components (`components/recording/`) +| Component | Purpose | +|-----------|---------| +| `audio-level-meter.tsx` | Real-time VU meter visualization | +| `confidence-indicator.tsx` | ASR confidence display | +| `vad-indicator.tsx` | Voice activity indicator | +| `buffering-indicator.tsx` | Congestion/buffering display | +| `recording-header.tsx` | Recording session header | +| `stat-card.tsx` | Statistics display | +| `speaker-distribution.tsx` | Speaker time breakdown | +| `idle-state.tsx` | Idle/standby UI | + +### Settings Components (`components/settings/`) +- `developer-options-section.tsx` +- `quick-actions-section.tsx` + +### Project Components (`components/projects/`) +- `ProjectScopeFilter.tsx` + +### Status & Badge Components +| Component | Purpose | +|-----------|---------| +| `entity-highlight.tsx` | NER entity highlighting | +| `entity-management-panel.tsx` | Entity CRUD UI | +| `annotation-type-badge.tsx` | Annotation type display | +| `meeting-state-badge.tsx` | Meeting state indicator | +| `priority-badge.tsx` | Priority indicator | +| `speaker-badge.tsx` | Speaker identification | +| `processing-status.tsx` | Post-processing indicator | +| `api-mode-indicator.tsx` | Connection mode indicator | +| `offline-banner.tsx` | Offline mode warning | + +### Layout Components +- `app-layout.tsx` — Main app shell with sidebar +- `app-sidebar.tsx` — Navigation sidebar +- `error-boundary.tsx` — React error boundary +- `empty-state.tsx` — Empty state template +- `meeting-card.tsx` — Meeting item card +- `NavLink.tsx` — Navigation link wrapper + +### Integration Components +- `calendar-connection-panel.tsx` — Calendar OAuth setup +- `calendar-events-panel.tsx` — Calendar events list +- `integration-config-panel/` — Integration setup wizard + +### Other Components +- `connection-status.tsx` — Connection status display +- `confirmation-dialog.tsx` — Generic confirm dialog +- `timestamped-notes-editor.tsx` — Annotation editor +- `preferences-sync-bridge.tsx` — Preferences sync coordinator +- `preferences-sync-status.tsx` — Sync status display + +## Pages (`pages/`) + +| Page | Path | Purpose | +|------|------|---------| +| `Home.tsx` | `/` | Landing/onboarding | +| `Recording.tsx` | `/recording` | Active recording session | +| `Meetings.tsx` | `/meetings` | Meetings list with filters | +| `MeetingDetail.tsx` | `/meetings/:id` | Meeting transcript & details | +| `Projects.tsx` | `/projects` | Project list & create | +| `ProjectSettings.tsx` | `/projects/:id/settings` | Project configuration | +| `Settings.tsx` | `/settings` | Application settings | +| `People.tsx` | `/people` | Workspace members | +| `Tasks.tsx` | `/tasks` | Action items/tasks view | +| `Analytics.tsx` | `/analytics` | Application analytics/logs | +| `NotFound.tsx` | `/*` | 404 fallback | + +### Settings Sub-Pages (`pages/settings/`) +- `IntegrationsTab.tsx` — Integration management +- `StatusTab.tsx` — System status + +### Meeting Detail Sub-Pages (`pages/meeting-detail/`) +Meeting detail components and sub-views. + +## Page Integration Patterns + +```typescript +// Recording.tsx example +export default function Recording() { + const { isConnected, isReadOnly } = useConnection(); + const { activeProject } = useProjects(); + const { state, start, cancel } = useDiarization(); + const { data: audioDevices } = useAudioDevices(); + + // Conditional rendering based on connection state + if (isReadOnly) return ; + + return ( + + + + + + ); +} +``` + +## Component Exports + +| Component | Purpose | +|-----------|---------| +| `` | Wraps app with connection state | +| `` | Wraps app with workspace state | +| `` | Wraps app with project state | +| `` | Main app shell with sidebar | +| `` | Real-time audio VU meter | +| `` | Recording session metadata | +| `` | Inline NER entity highlighting | +| `` | Action item/decision/risk badge | +| `` | Meeting state indicator | +| `` | Cached/offline mode warning | +| `` | Post-processing progress | +| `` | Connection mode display | + +## Analytics Components (`components/analytics/`) +- `logs-tab.tsx` — Log viewer +- `performance-tab.tsx` — Performance metrics +- `analytics-utils.ts` — Analytics utilities diff --git a/.rag/10-rust-tauri-commands.md b/.rag/10-rust-tauri-commands.md new file mode 100644 index 0000000..6dfb2dd --- /dev/null +++ b/.rag/10-rust-tauri-commands.md @@ -0,0 +1,238 @@ +# NoteFlow Rust/Tauri Commands + +## Location +`client/src-tauri/src/commands/` + +## Command Summary (97 Total) + +### Connection (5) +| Command | Purpose | +|---------|---------| +| `connect()` | Connect to gRPC server | +| `disconnect()` | Disconnect from server | +| `is_connected()` | Check connection status | +| `get_server_info()` | Get server info | +| `get_effective_server_url()` | Get resolved server URL | + +### Identity (8) +| Command | Purpose | +|---------|---------| +| `get_current_user()` | Get current user | +| `list_workspaces()` | List available workspaces | +| `switch_workspace()` | Switch active workspace | +| `initiate_auth_login()` | Start auth flow | +| `complete_auth_login()` | Complete auth flow | +| `logout()` | Log out | +| `get_workspace_settings()` | Get workspace settings | +| `update_workspace_settings()` | Update workspace settings | + +### Projects (16) +| Command | Purpose | +|---------|---------| +| `create_project()` | Create new project | +| `get_project()` | Get project by ID | +| `list_projects()` | List all projects | +| `update_project()` | Update project | +| `set_active_project()` | Set active project | +| `add_project_member()` | Add member | +| `update_project_member_role()` | Update member role | +| `remove_project_member()` | Remove member | +| `list_project_members()` | List members | +| `archive_project()` | Archive project | +| `restore_project()` | Restore project | +| `delete_project()` | Delete project | + +### Meeting (5) +| Command | Purpose | +|---------|---------| +| `create_meeting()` | Create meeting | +| `list_meetings()` | List meetings | +| `get_meeting()` | Get meeting by ID | +| `stop_meeting()` | Stop recording | +| `delete_meeting()` | Delete meeting | + +### Recording (5) — `recording/` +| Command | Purpose | +|---------|---------| +| `start_recording()` | Start recording session | +| `stop_recording()` | Stop recording session | +| `send_audio_chunk()` | Stream audio samples | +| `get_stream_state()` | Get stream state | +| `reset_stream_state()` | Reset stream state | + +**Recording Module Files**: +- `session/mod.rs` — Session lifecycle +- `session/chunks/mod.rs` — Audio chunk streaming +- `capture.rs` — Native audio capture (cpal) +- `device.rs` — Device resolution utilities +- `dual_capture.rs` — Mic + system audio mixing +- `stream_state.rs` — VU levels, RMS, counts +- `app_policy.rs` — Recording app policy + +### Annotation (5) +| Command | Purpose | +|---------|---------| +| `add_annotation()` | Add annotation | +| `get_annotation()` | Get annotation | +| `list_annotations()` | List annotations | +| `update_annotation()` | Update annotation | +| `delete_annotation()` | Delete annotation | + +### Summary (9) +| Command | Purpose | +|---------|---------| +| `generate_summary()` | Generate summary | +| `list_summarization_templates()` | List templates | +| `create_summarization_template()` | Create template | +| `update_summarization_template()` | Update template | +| `delete_summarization_template()` | Delete template | +| `get_summarization_template()` | Get template | +| `grant_cloud_consent()` | Grant cloud AI consent | +| `revoke_cloud_consent()` | Revoke consent | +| `get_cloud_consent_status()` | Check consent | + +### Export (2) +| Command | Purpose | +|---------|---------| +| `export_transcript()` | Export to format | +| `save_export_file()` | Save export to disk | + +### Entities (3) +| Command | Purpose | +|---------|---------| +| `extract_entities()` | Run NER extraction | +| `update_entity()` | Update entity | +| `delete_entity()` | Delete entity | + +### Diarization (5) +| Command | Purpose | +|---------|---------| +| `refine_speaker_diarization()` | Start refinement | +| `get_diarization_job_status()` | Get job status | +| `rename_speaker()` | Rename speaker | +| `cancel_diarization_job()` | Cancel job | +| `get_active_diarization_jobs()` | List active jobs | + +### Audio Devices (12) — `audio.rs` +| Command | Purpose | +|---------|---------| +| `list_audio_devices()` | List input/output devices | +| `get_default_audio_device()` | Get system default | +| `select_audio_device()` | Select device | +| `start_input_test()` | Start input test | +| `stop_input_test()` | Stop input test | +| `list_loopback_devices()` | List system audio devices | +| `set_system_audio_device()` | Set system audio capture | +| `set_dual_capture_enabled()` | Enable mic + system | +| `set_audio_mix_levels()` | Set mix levels | +| `get_dual_capture_config()` | Get dual capture config | + +### Playback (5) — `playback/` +| Command | Purpose | +|---------|---------| +| `start_playback()` | Start audio playback | +| `pause_playback()` | Pause playback | +| `stop_playback()` | Stop playback | +| `seek_playback()` | Seek position | +| `get_playback_state()` | Get playback state | + +### Preferences (4) +| Command | Purpose | +|---------|---------| +| `get_preferences()` | Get user preferences | +| `save_preferences()` | Save preferences | +| `get_preferences_sync()` | Get sync preferences | +| `set_preferences_sync()` | Set sync preferences | + +### Triggers (6) — `triggers/` +| Command | Purpose | +|---------|---------| +| `set_trigger_enabled()` | Enable/disable triggers | +| `snooze_triggers()` | Snooze triggers | +| `reset_snooze()` | Reset snooze | +| `get_trigger_status()` | Get trigger status | +| `dismiss_trigger()` | Dismiss trigger | +| `accept_trigger()` | Accept trigger | + +### Calendar (7) +| Command | Purpose | +|---------|---------| +| `list_calendar_events()` | List events | +| `initiate_oauth()` | Start OAuth flow | +| `initiate_oauth_loopback()` | OAuth with loopback | +| `complete_oauth()` | Complete OAuth | +| `get_oauth_connection_status()` | Check connection | +| `disconnect_oauth()` | Disconnect OAuth | +| `get_calendar_providers()` | List providers | + +### Webhooks (5) +| Command | Purpose | +|---------|---------| +| `register_webhook()` | Register webhook | +| `list_webhooks()` | List webhooks | +| `update_webhook()` | Update webhook | +| `delete_webhook()` | Delete webhook | +| `get_webhook_deliveries()` | Get delivery history | + +### OIDC (8) +| Command | Purpose | +|---------|---------| +| `register_oidc_provider()` | Register provider | +| `list_oidc_providers()` | List providers | +| `get_oidc_provider()` | Get provider | +| `update_oidc_provider()` | Update provider | +| `delete_oidc_provider()` | Delete provider | +| `refresh_oidc_discovery()` | Refresh discovery | +| `test_oidc_connection()` | Test connection | +| `list_oidc_presets()` | List presets | + +### Integration Sync (4) +| Command | Purpose | +|---------|---------| +| `start_integration_sync()` | Start sync | +| `get_sync_status()` | Get sync status | +| `list_sync_history()` | List sync history | +| `get_user_integrations()` | Get integrations | + +### Observability (2) +| Command | Purpose | +|---------|---------| +| `get_recent_logs()` | Get recent logs | +| `get_performance_metrics()` | Get metrics | + +### ASR & Streaming Config (5) +| Command | Purpose | +|---------|---------| +| `get_asr_configuration()` | Get ASR config | +| `update_asr_configuration()` | Update ASR config | +| `get_asr_job_status()` | Get ASR job status | +| `get_streaming_configuration()` | Get streaming config | +| `update_streaming_configuration()` | Update streaming config | + +### HuggingFace (4) +| Command | Purpose | +|---------|---------| +| `set_huggingface_token()` | Set token | +| `get_huggingface_token_status()` | Get token status | +| `delete_huggingface_token()` | Delete token | +| `validate_huggingface_token()` | Validate token | + +### Apps (2) +| Command | Purpose | +|---------|---------| +| `list_installed_apps()` | List apps | +| `invalidate_app_cache()` | Clear app cache | + +### Diagnostics & Testing (5) +| Command | Purpose | +|---------|---------| +| `run_connection_diagnostics()` | Run diagnostics | +| `check_test_environment()` | Check test env | +| `reset_test_recording_state()` | Reset test state | +| `inject_test_audio()` | Inject test audio | +| `inject_test_tone()` | Inject test tone | + +### Shell (1) +| Command | Purpose | +|---------|---------| +| `open_url()` | Open URL in browser | diff --git a/.rag/11-rust-grpc-types.md b/.rag/11-rust-grpc-types.md new file mode 100644 index 0000000..4afd07e --- /dev/null +++ b/.rag/11-rust-grpc-types.md @@ -0,0 +1,191 @@ +# NoteFlow Rust gRPC Client & Types + +## Location +`client/src-tauri/src/grpc/` + +## Architecture +Modular client using trait extensions for composition. + +```rust +pub struct GrpcClient { + channel: Channel, + identity: Arc, + state: Arc, +} +``` + +## Type Definitions (`grpc/types/`) + +### Core Types (`core.rs`) + +```rust +pub struct ServerInfo { + pub version: String, + pub asr_model: String, + pub asr_ready: bool, + pub supported_sample_rates: Vec, + pub max_chunk_size: i32, + pub uptime_seconds: f64, + pub active_meetings: i32, + pub diarization_enabled: bool, + pub diarization_ready: bool, + pub state_version: i64, + pub system_ram_total_bytes: Option, + pub gpu_vram_total_bytes: Option, +} + +pub struct Meeting { + pub id: String, + pub project_id: Option, + pub title: String, + pub state: MeetingState, + pub created_at: f64, + pub started_at: Option, + pub ended_at: Option, + pub duration_seconds: f64, + pub segments: Vec, + pub summary: Option, + pub metadata: HashMap, +} + +pub struct Segment { + pub id: String, + pub speaker: String, + pub text: String, + pub start_time: f64, + pub end_time: f64, + pub confidence: f32, + pub speaker_id: Option, +} + +pub struct Summary { + pub id: String, + pub content: String, + pub template_id: Option, + pub generated_at: f64, + pub created_by_ai: bool, +} + +pub struct Annotation { + pub id: String, + pub segment_ids: Vec, + pub annotation_type: AnnotationType, + pub content: String, + pub created_at: f64, +} +``` + +### Enums (`enums.rs`) + +```rust +pub enum MeetingState { + Unspecified = 0, + Created = 1, + Recording = 2, + Stopped = 3, + Completed = 4, + Error = 5, +} + +pub enum AnnotationType { + Unspecified = 0, + ActionItem = 1, + Decision = 2, + Note = 3, + Risk = 4, +} + +pub enum ExportFormat { + Unspecified = 0, + Markdown = 1, + Html = 2, + Pdf = 3, +} + +pub enum UpdateType { + Unspecified = 0, + PartialTranscript = 1, + FinalTranscript = 2, + SpeakerDiarization = 3, +} + +pub enum Priority { + Unspecified = 0, + Low = 1, + Medium = 2, + High = 3, +} +``` + +### Other Type Modules +- `asr.rs` — ASR config types +- `streaming.rs` — Streaming types +- `projects.rs` — Project types +- `calendar.rs` — Calendar types +- `webhooks.rs` — Webhook types +- `preferences.rs` — Preference types +- `identity.rs` — Identity types +- `oidc.rs` — OIDC types +- `sync.rs` — Sync types +- `observability.rs` — Observability types +- `results.rs` — Result wrappers +- `hf_token.rs` — HuggingFace types + +## Client Modules (`grpc/client/`) + +| File | Purpose | +|------|---------| +| `core.rs` | Connection, auth, identity interceptor | +| `meetings.rs` | Meeting CRUD | +| `annotations.rs` | Annotation ops | +| `diarization.rs` | Diarization requests | +| `identity.rs` | User/workspace ops | +| `projects.rs` | Project ops | +| `preferences.rs` | Preference ops | +| `calendar.rs` | Calendar sync | +| `webhooks.rs` | Webhook ops | +| `oidc.rs` | OIDC providers | +| `sync.rs` | Integration sync | +| `observability.rs` | Logs/metrics | +| `asr.rs` | ASR config | +| `streaming.rs` | Streaming config | +| `hf_token.rs` | HuggingFace tokens | +| `converters.rs` | Proto ↔ domain converters | + +## Streaming (`grpc/streaming/`) + +```rust +pub struct StreamManager { + state: Arc, + grpc_client: Arc, +} + +pub struct AudioStreamChunk { + pub segment_id: String, + pub samples: Vec, + pub timestamp: f64, + pub speaker: Option, +} + +pub struct StreamStateInfo { + pub is_streaming: bool, + pub buffered_samples: usize, + pub segments_completed: u32, +} +``` + +## Identity Interceptor + +```rust +impl Interceptor for IdentityInterceptor { + fn call(&mut self, mut request: Request<()>) -> Result, Status> { + let metadata = request.metadata_mut(); + metadata.insert("x-user-id", user_id.parse()?); + metadata.insert("x-workspace-id", workspace_id.parse()?); + if let Some(token) = self.identity.access_token() { + metadata.insert("authorization", format!("Bearer {token}").parse()?); + } + Ok(request) + } +} +``` diff --git a/.rag/12-rust-state-audio-crypto.md b/.rag/12-rust-state-audio-crypto.md new file mode 100644 index 0000000..51013f1 --- /dev/null +++ b/.rag/12-rust-state-audio-crypto.md @@ -0,0 +1,307 @@ +# NoteFlow Rust State, Audio & Crypto + +## Location +`client/src-tauri/src/state/`, `audio/`, `crypto/` + +## State Management (`state/`) + +### AppState (`app_state.rs`) + +```rust +pub struct AppState { + // Connection State + pub grpc_client: Arc, + pub server_info: RwLock>, + + // Recording State + pub recording: RwLock>, + pub current_meeting: RwLock>, + pub recording_start_time: RwLock>, + pub elapsed_seconds: RwLock, + + // Playback State + pub playback_handle: Arc, + pub playback_state: RwLock, + pub playback_info: RwLock>, + + // Audio Configuration + pub audio_config: RwLock, + + // Trigger State + pub trigger_status: RwLock, + pub pending_triggers: RwLock>, + pub dismissed_triggers: RwLock>, + + // User Preferences + pub preferences: RwLock, + + // Crypto (lazy-initialized) + pub crypto: CryptoManager, + + // Identity (lazy-initialized) + pub identity: Arc, +} +``` + +### Recording Session + +```rust +pub struct RecordingSession { + pub meeting_id: String, + pub start_time: f64, + pub audio_tx: mpsc::Sender, + pub mic_input_device: Option, + pub system_audio_device: Option, +} + +pub struct AudioSamplesChunk { + pub samples: Vec, + pub timestamp: f64, + pub sample_rate: u32, + pub channels: u16, +} +``` + +### Preferences + +```rust +pub struct UserPreferences { + pub default_export_location: String, + pub audio_devices: AudioDevicePrefs, + pub playback_volume: f32, + pub auto_play_on_load: bool, + pub app_policies: Vec, +} + +pub struct AudioConfig { + pub input_device_id: Option, + pub output_device_id: Option, + pub system_device_id: Option, + pub dual_capture_enabled: bool, + pub mic_gain: f32, + pub system_gain: f32, +} +``` + +## Audio Handling (`audio/`) + +### Audio Capture (`capture.rs`) + +```rust +pub struct CaptureConfig { + pub sample_rate: u32, + pub channels: u16, + pub buffer_size: usize, +} + +pub struct AudioCapture { + stream: Stream, // cpal Stream +} + +impl AudioCapture { + pub fn new( + config: CaptureConfig, + audio_tx: mpsc::Sender>, + level_callback: Arc, + ) -> Result; + + pub fn with_device( + device: Device, + config: CaptureConfig, + audio_tx: mpsc::Sender>, + level_callback: Arc, + ) -> Result; + + pub fn pause() -> Result<()>; + pub fn resume() -> Result<()>; +} +``` + +### Audio Playback (`playback.rs`) + +```rust +pub enum PlaybackCommand { + Play(Vec, u32), // audio buffer, sample rate + Pause, + Resume, + Stop, + Shutdown, +} + +pub struct PlaybackHandle { + command_tx: Sender, + response_rx: Mutex>>, + _thread: JoinHandle<()>, +} + +pub struct PlaybackStarted { + pub duration: f64, +} +``` + +### Device Management (`devices.rs`) +- `list_audio_devices()` — Enumerate devices +- `get_default_audio_device()` — System default +- `select_audio_device()` — User selection +- `get_supported_configs()` — Device capabilities + +### Utilities +- `mixer.rs` — Dual-capture audio mixing +- `loader.rs` — Audio file loading +- `windows_loopback.rs` — Windows system audio capture + +## Encryption (`crypto/`) + +### CryptoBox (AES-256-GCM) + +```rust +pub struct CryptoBox { + cipher: Aes256Gcm, +} + +pub struct CryptoManager { + crypto: OnceLock>, +} + +impl CryptoBox { + pub fn new() -> Result; // Create with keychain key + pub fn with_key(key: &[u8; 32]) -> Result; + pub fn encrypt_file(input: &Path, output: &Path) -> Result<()>; + pub fn decrypt_file(input: &Path, output: &Path) -> Result<()>; +} + +impl CryptoManager { + pub fn get_or_init(&self) -> Result<&CryptoBox>; + pub fn encrypt_file_async(&self, input: PathBuf, output: PathBuf) -> JoinHandle>; +} +``` + +**File Format**: `[4-byte magic: NFAE] [16-byte nonce] [ciphertext] [16-byte tag]` + +## Identity (`identity/`) + +```rust +pub struct StoredIdentity { + pub user_id: String, + pub workspace_id: String, + pub display_name: String, + pub email: Option, + pub workspace_name: String, + pub role: String, + pub is_local: bool, +} + +pub struct IdentityManager { + identity: OnceLock, + access_token: OnceLock>, +} + +impl IdentityManager { + pub fn new() -> Self; + pub fn is_authenticated(&self) -> bool; + pub fn user_id(&self) -> String; + pub fn workspace_id(&self) -> String; + pub fn access_token(&self) -> Option; + pub fn store_identity(&self, identity: StoredIdentity) -> Result<()>; + pub fn store_access_token(&self, token: String) -> Result<()>; +} +``` + +**Features**: +- Local-first default: `StoredIdentity::local_default()` +- Keychain storage for persistent identity +- Lazy initialization to defer keychain access + +## Error Handling (`error/`) + +```rust +pub enum Error { + Grpc(Box), + GrpcTransport(Box), + Connection(String), + AudioCapture(String), + AudioPlayback(String), + Encryption(String), + Io(std::io::Error), + Serialization(serde_json::Error), + NotConnected, + AlreadyConnected, + NoActiveRecording, + AlreadyRecording, + NoActivePlayback, + AlreadyPlaying, + MeetingNotFound(String), + AnnotationNotFound(String), + IntegrationNotFound(String), + DeviceNotFound(String), + InvalidOperation(String), + InvalidInput(String), + Stream(String), + Timeout(String), +} + +pub struct ErrorClassification { + pub grpc_status: Option, + pub category: String, // network, auth, validation, not_found, server, client + pub retryable: bool, +} +``` + +## Trigger Detection (`triggers/`) + +### Foreground App Detection + +```rust +pub struct ForegroundAppIdentity { + pub name: String, + pub bundle_id: Option, // macOS + pub app_id: Option, // Windows + pub exe_path: Option, + pub exe_name: Option, + pub desktop_id: Option, // Linux + pub is_pwa: bool, +} + +pub fn detect_foreground_app() -> Option; +pub fn is_meeting_app(app: &ForegroundAppIdentity) -> bool; +``` + +**Meeting Apps**: zoom, teams, meet, slack, webex, discord, skype, gotomeeting, facetime, ringcentral + +### Trigger Signals + +```rust +pub struct TriggerSignal { + pub source: TriggerSource, // ForegroundApp, AudioActivity, CalendarProximity + pub confidence: f32, // 0.0-1.0 + pub action: TriggerAction, // Ignore, Notify, AutoStart + pub timestamp: f64, +} + +pub struct PendingTrigger { + pub id: String, + pub signal: TriggerSignal, + pub dismissed_until: Option, +} +``` + +## Events (`events/`) + +```rust +pub enum AppEvent { + Connected { server_info: ServerInfo }, + Disconnected { reason: String }, + ConnectionError { error: String }, + RecordingStarted { meeting_id: String }, + RecordingProgress { elapsed_seconds: u32 }, + RecordingStopped { meeting_id: String }, + PlaybackStarted { duration: f64 }, + PlaybackStopped, + PlaybackProgress { current_position: f64 }, + TriggerDetected { signal: TriggerSignal }, + TriggerDismissed { trigger_id: String }, + AudioLevelChanged { rms: f32 }, + AudioActivityDetected, + Error { error: String, category: String }, +} +``` diff --git a/.rag/13-common-utilities.md b/.rag/13-common-utilities.md new file mode 100644 index 0000000..d4af528 --- /dev/null +++ b/.rag/13-common-utilities.md @@ -0,0 +1,196 @@ +# NoteFlow Common Utilities & Patterns + +## Python Utilities + +### Domain Utilities (`domain/utils/`) +- `time.py` — `utc_now()` for UTC-aware datetime +- `validation.py` — Validation helpers + +### Config Constants (`config/constants/`) +- `core.py` — DAYS_PER_WEEK, HOURS_PER_DAY, DEFAULT_LLM_TEMPERATURE +- `domain.py` — DEFAULT_ANTHROPIC_MODEL, WEBHOOK_EVENT_TYPES +- `errors.py` — Error code constants +- `http.py` — HTTP-related constants + +### Domain Constants (`domain/constants/`) +- `fields.py` — Field name constants (EMAIL, PROVIDER, PROJECT_ID) + +### Settings (`config/settings/`) +- `_main.py` — Main settings (database, ASR, gRPC, storage) +- `_triggers.py` — Trigger settings +- `_calendar.py` — Calendar provider settings +- `_features.py` — Feature flags + +**Key Env Vars**: +- `NOTEFLOW_DATABASE_URL` — PostgreSQL connection +- `NOTEFLOW_ASR_MODEL_SIZE` — Whisper size +- `NOTEFLOW_GRPC_PORT` — gRPC port (default: 50051) +- `NOTEFLOW_MEETINGS_DIR` — Audio storage directory +- `NOTEFLOW_RETENTION_DAYS` — Retention policy (default: 90) + +## TypeScript Utilities (`client/src/lib/`) + +### Configuration (`config/`) +- `app-config.ts` — App-wide constants +- `defaults.ts` — Default values +- `provider-endpoints.ts` — AI provider endpoints +- `server.ts` — Server connection defaults + +### Formatting & Time +- `format.ts` — Time, duration, GB, percentages +- `time.ts` — Time constants (SECONDS_PER_*, MS_PER_*) +- `timing-constants.ts` — Polling intervals, timeouts + +### Logging +```typescript +// NEVER use console.log - always use clientlog system +import { addClientLog } from '@/lib/client-logs'; +import { debug, errorLog } from '@/lib/debug'; + +addClientLog({ + level: 'info', + source: 'app', + message: 'Event occurred', + details: 'Context', +}); + +const log = debug('ComponentName'); +log('Debug message', { data }); + +const logError = errorLog('ComponentName'); +logError('Error occurred', error); +``` + +### Preferences (`preferences/`) +```typescript +import { preferences } from '@/lib/preferences'; + +// Initialize on app start +await preferences.initialize(); + +// Read/write +const prefs = preferences.get(); +await preferences.set('selected_input_device', deviceId); +await preferences.replace({ ...current, theme: 'dark' }); + +// Subscribe to changes +const unsubscribe = preferences.subscribe((prefs) => { + console.log('Changed:', prefs); +}); +``` + +### Storage +- `storage-keys.ts` — localStorage key constants +- `storage-utils.ts` — localStorage helpers + +### Caching +- `cache/meeting-cache.ts` — Meeting cache with TTL & events + +### Event System +- `tauri-events.ts` — Tauri event subscriptions +- `event-emitter.ts` — Generic event emitter + +### Other Utilities +- `utils.ts` — Generic TypeScript utilities +- `object-utils.ts` — Object manipulation +- `speaker-utils.ts` — Speaker ID formatting +- `integration-utils.ts` — Integration helpers +- `entity-store.ts` — Entity caching for NER +- `crypto.ts` — Client-side encryption +- `oauth-utils.ts` — OAuth flow helpers + +## Rust Constants (`constants.rs`) + +```rust +pub mod grpc { + pub const CONNECTION_TIMEOUT: Duration = Duration::from_secs(5); + pub const REQUEST_TIMEOUT: Duration = Duration::from_secs(300); + pub const DEFAULT_PORT: u16 = 50051; + pub const MAX_RETRY_ATTEMPTS: u32 = 3; +} + +pub mod audio { + pub const DEFAULT_SAMPLE_RATE: u32 = 16000; + pub const DEFAULT_CHANNELS: u32 = 1; + pub const DEFAULT_BUFFER_SIZE: usize = 1600; // 100ms + pub const MIN_DB_LEVEL: f32 = -60.0; + pub const MAX_DB_LEVEL: f32 = 0.0; +} + +pub mod storage { + pub const MAX_AUDIO_SIZE_BYTES: u64 = 5 * 1024 * 1024 * 1024; // 5GB +} + +pub mod triggers { + pub const POLL_INTERVAL: Duration = Duration::from_secs(5); + pub const DEFAULT_SNOOZE_DURATION: Duration = Duration::from_secs(600); + pub const AUTO_START_THRESHOLD: f32 = 0.7; +} + +pub mod crypto { + pub const KEY_SIZE: usize = 32; // 256-bit AES + pub const KEYCHAIN_SERVICE: &str = "NoteFlow"; +} + +pub mod identity { + pub const DEFAULT_USER_ID: &str = "local-user"; + pub const DEFAULT_WORKSPACE_ID: &str = "local-workspace"; +} +``` + +## Key Patterns + +### Python: Protocol-Based Dependency Injection +```python +class MeetingService: + def __init__(self, uow: UnitOfWork): + self._uow = uow + + async def create_meeting(self, params: MeetingCreateParams) -> Meeting: + async with self._uow: + meeting = Meeting.create(params) + await self._uow.meetings.add(meeting) + await self._uow.commit() + return meeting +``` + +### TypeScript: Connection-Aware Components +```typescript +function MyComponent() { + const { isConnected, isReadOnly } = useConnection(); + + if (isReadOnly) return ; + return ; +} +``` + +### Rust: Lazy Initialization +```rust +pub struct CryptoManager { + crypto: OnceLock>, +} + +impl CryptoManager { + pub fn get_or_init(&self) -> Result<&CryptoBox> { + self.crypto.get_or_try_init(CryptoBox::new) + } +} +``` + +### Rust: State Access via RwLock +```rust +let mut rec = state.recording.write(); +*rec = Some(session); +``` + +## Feature Flags + +| Flag | Default | Controls | +|------|---------|----------| +| `NOTEFLOW_FEATURE_TEMPLATES_ENABLED` | `true` | AI templates | +| `NOTEFLOW_FEATURE_PDF_EXPORT_ENABLED` | `true` | PDF export | +| `NOTEFLOW_FEATURE_NER_ENABLED` | `false` | Entity extraction | +| `NOTEFLOW_FEATURE_CALENDAR_ENABLED` | `false` | Calendar sync | +| `NOTEFLOW_FEATURE_WEBHOOKS_ENABLED` | `true` | Webhooks | + +Access: `get_feature_flags().` or `get_settings().feature_flags.` diff --git a/CLAUDE.md b/CLAUDE.md index 49b5cd2..b1be8f2 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -2,6 +2,191 @@ This file provides guidance to Claude Code when working with code in this repository. +--- + +## 🚨 CRITICAL: PARALLEL EXECUTION AFTER SWARM INIT + +**MANDATORY RULE**: Once swarm is initialized with memory, ALL subsequent operations MUST be parallel: + +- **TodoWrite** → Always batch 5-10+ todos in ONE call +- **Task spawning** → Spawn ALL agents in ONE message +- **File operations** → Batch ALL reads/writes together +- **NEVER** operate sequentially after swarm init + +--- + +## 🚨 CRITICAL: CONCURRENT EXECUTION FOR ALL ACTIONS + +**ABSOLUTE RULE**: ALL operations MUST be concurrent/parallel in a single message: + +### 🔴 MANDATORY CONCURRENT PATTERNS: + +- **TodoWrite**: ALWAYS batch ALL todos in ONE call (5-10+ todos minimum) +- **Task tool**: ALWAYS spawn ALL agents in ONE message with full instructions +- **File operations**: ALWAYS batch ALL reads/writes/edits in ONE message +- **Bash commands**: ALWAYS batch ALL terminal operations in ONE message +- **Memory operations**: ALWAYS batch ALL memory store/retrieve in ONE message + +### ⚡ GOLDEN RULE: "1 MESSAGE = ALL RELATED OPERATIONS" + +**Examples of CORRECT concurrent execution:** + +```bash +# ✅ CORRECT: Everything in ONE message +TodoWrite { todos: [10+ todos with all statuses/priorities] } +Task("Agent 1 with full instructions and hooks") +Task("Agent 2 with full instructions and hooks") +Task("Agent 3 with full instructions and hooks") +Read("file1.js") +Read("file2.js") +Read("file3.js") +Write("output1.js", content) +Write("output2.js", content) +Bash("npm install") +Bash("npm test") +Bash("npm run build") +``` + +**Examples of WRONG sequential execution:** + +```bash +# ❌ WRONG: Multiple messages (NEVER DO THIS) +Message 1: TodoWrite { todos: [single todo] } +Message 2: Task("Agent 1") +Message 3: Task("Agent 2") +Message 4: Read("file1.js") +Message 5: Write("output1.js") +Message 6: Bash("npm install") +# This is 6x slower and breaks coordination! +``` + +### 🎯 CONCURRENT EXECUTION CHECKLIST: + +Before sending ANY message, ask yourself: + +- ✅ Are ALL related TodoWrite operations batched together? +- ✅ Are ALL Task spawning operations in ONE message? +- ✅ Are ALL file operations (Read/Write/Edit) batched together? +- ✅ Are ALL bash commands grouped in ONE message? +- ✅ Are ALL memory operations concurrent? + +If ANY answer is "No", you MUST combine operations into a single message! + +--- + +## 🚀 CRITICAL: Claude Code Does ALL Real Work + +### 🎯 CLAUDE CODE IS THE ONLY EXECUTOR + +**ABSOLUTE RULE**: Claude Code performs ALL actual work: + +✅ **Claude Code ALWAYS Handles:** +- 🔧 ALL file operations (Read, Write, Edit, MultiEdit, Glob, Grep) +- 💻 ALL code generation and programming tasks +- 🖥️ ALL bash commands and system operations +- 🏗️ ALL actual implementation work +- 🔍 ALL project navigation and code analysis +- 📝 ALL TodoWrite and task management +- 🔄 ALL git operations (commit, push, merge) +- 📦 ALL package management (npm, pip, etc.) +- 🧪 ALL testing and validation +- 🔧 ALL debugging and troubleshooting + +--- + +## 🚀 CRITICAL: Parallel Execution & Batch Operations + +### 🚨 MANDATORY RULE #1: BATCH EVERYTHING + +When using swarms, you MUST use BatchTool for ALL operations: + +- **NEVER** send multiple messages for related operations +- **ALWAYS** combine multiple tool calls in ONE message +- **PARALLEL** execution is MANDATORY, not optional + +### ⚡ THE GOLDEN RULE OF SWARMS + +If you need to do X operations, they should be in 1 message, not X messages + +### 🚨 MANDATORY TODO AND TASK BATCHING + +**CRITICAL RULE FOR TODOS AND TASKS:** + +- **TodoWrite** MUST ALWAYS include ALL todos in ONE call (5-10+ todos) +- **Task** tool calls MUST be batched - spawn multiple agents in ONE message +- **NEVER** update todos one by one - this breaks parallel coordination +- **NEVER** spawn agents sequentially - ALL agents spawn together + +### 📦 BATCH TOOL EXAMPLES + +**✅ CORRECT - Everything in ONE Message:** + +```bash +# Single Message with BatchTool +# MCP coordination setup +mcp__claude-flow__swarm_init { topology: "mesh", maxAgents: 6 } +mcp__claude-flow__agent_spawn { type: "researcher" } +mcp__claude-flow__agent_spawn { type: "coder" } +mcp__claude-flow__agent_spawn { type: "analyst" } +mcp__claude-flow__agent_spawn { type: "tester" } +mcp__claude-flow__agent_spawn { type: "coordinator" } + +# Claude Code execution - ALL in parallel +Task("You are researcher agent. MUST coordinate via hooks...") +Task("You are coder agent. MUST coordinate via hooks...") +Task("You are analyst agent. MUST coordinate via hooks...") +Task("You are tester agent. MUST coordinate via hooks...") +TodoWrite { todos: [5-10 todos with all priorities and statuses] } + +# File operations in parallel +Bash "mkdir -p app/{src,tests,docs}" +Write "app/package.json" +Write "app/README.md" +Write "app/src/index.js" +``` + +**❌ WRONG - Multiple Messages (NEVER DO THIS):** + +```bash +# This is 6x slower and breaks parallel coordination! +Message 1: mcp__claude-flow__swarm_init +Message 2: Task("researcher agent") +Message 3: Task("coder agent") +Message 4: TodoWrite({ todo: "single todo" }) +Message 5: Bash "mkdir src" +Message 6: Write "package.json" +``` + +### 🎯 BATCH OPERATIONS BY TYPE + +**Todo and Task Operations (Single Message):** + +- **TodoWrite** → ALWAYS include 5-10+ todos in ONE call +- **Task agents** → Spawn ALL agents with full instructions in ONE message +- **Agent coordination** → ALL Task calls must include coordination hooks +- **Status updates** → Update ALL todo statuses together +- **NEVER** split todos or Task calls across messages! + +**File Operations (Single Message):** + +- **Read 10 files?** → One message with 10 Read calls +- **Write 5 files?** → One message with 5 Write calls +- **Edit 1 file many times?** → One MultiEdit call + +**Swarm Operations (Single Message):** + +- **Need 8 agents?** → One message with swarm_init + 8 agent_spawn calls +- **Multiple memories?** → One message with all memory_usage calls +- **Task + monitoring?** → One message with task_orchestrate + swarm_monitor + +**Command Operations (Single Message):** + +- **Multiple directories?** → One message with all mkdir commands +- **Install + test + lint?** → One message with all npm commands +- **Git operations?** → One message with all git commands + +--- + ## Project Overview NoteFlow is an intelligent meeting notetaker: local-first audio capture + navigable recall + evidence-linked summaries. It is a client-server system built around a gRPC API for bidirectional audio streaming and transcription. The repository includes: @@ -26,28 +211,39 @@ The gRPC schema is the shared contract between backend and client; keep proto ch --- +## Specialized Documentation + +For detailed development guidance, see the specialized documentation files: + +- **Python Backend Development** → `src/noteflow/CLAUDE.md` +- **Client Development (TypeScript + Rust)** → `client/CLAUDE.md` + +--- + ## Build and Development Commands ```bash -# Install (editable with dev dependencies) +# Install Python backend (editable with dev dependencies) python -m pip install -e ".[dev]" +# Install client dependencies +cd client && npm install + # Run gRPC server python -m noteflow.grpc.server --help -# Run Tauri + React client UI -cd client && npm install && npm run dev -cd client && npm run tauri dev # Desktop (requires Rust) +# Run client UI +cd client && npm run dev # Web UI (Vite) +cd client && npm run tauri dev # Desktop (requires Rust) # Tests -pytest # Full suite -pytest -m "not integration" # Skip external-service tests -pytest tests/domain/ # Run specific test directory -pytest -k "test_segment" # Run by pattern +pytest # Full Python suite +cd client && npm run test # Client tests +cd client && npm run test:e2e # E2E tests # Docker (hot-reload enabled) -docker compose up -d postgres # PostgreSQL with pgvector -python scripts/dev_watch_server.py # Auto-reload server +docker compose up -d postgres # PostgreSQL with pgvector +python scripts/dev_watch_server.py # Auto-reload server ``` ### Forbidden Docker Operations (without explicit permission) @@ -62,890 +258,71 @@ python scripts/dev_watch_server.py # Auto-reload server **Always use Makefile targets instead of running tools directly.** -### Primary Quality Commands - ```bash make quality # Run ALL quality checks (TS + Rust + Python) make quality-py # Python: lint + type-check + test-quality make quality-ts # TypeScript: type-check + lint + test-quality make quality-rs # Rust: clippy + lint -``` - -### Python Quality - -```bash -make lint-py # Basedpyright lint → .hygeine/basedpyright.lint.json -make type-check-py # Basedpyright strict mode -make test-quality-py # pytest tests/quality/ -make lint-fix-py # Auto-fix Ruff + Sourcery issues -``` - -### TypeScript Quality - -```bash -make type-check # tsc --noEmit -make lint # Biome linter → .hygeine/biome.json -make lint-fix # Auto-fix Biome issues -make test-quality # Vitest quality tests -``` - -### Rust Quality - -```bash -make clippy # Clippy linter → .hygeine/clippy.json -make lint-rs # Code quality script → .hygeine/rust_code_quality.txt -make fmt-rs # Format with rustfmt -make fmt-check-rs # Check rustfmt formatting -``` - -### Formatting - -```bash make fmt # Format all code (Biome + rustfmt) -make fmt-check # Check all formatting -``` - -### E2E Tests - -```bash make e2e # Playwright tests (requires frontend on :5173) -make e2e-ui # Playwright tests with UI mode -make e2e-grpc # Rust gRPC integration tests ``` --- -## Architecture +## Repository Structure ``` -src/noteflow/ -├── domain/ # Entities, ports, value objects -│ ├── entities/ # Meeting, Segment, Summary, Annotation, NamedEntity, Integration, Project, Processing, SummarizationTemplate -│ ├── identity/ # User, Workspace, WorkspaceMembership, roles, context -│ ├── auth/ # OIDC discovery, claims, constants -│ ├── ports/ # Repository protocols -│ │ ├── repositories/ -│ │ │ ├── transcript.py # MeetingRepository, SegmentRepository, SummaryRepository -│ │ │ ├── asset.py # AssetRepository -│ │ │ ├── background.py # DiarizationJobRepository -│ │ │ ├── external/ # WebhookRepository, IntegrationRepository, EntityRepository, UsageRepository -│ │ │ └── identity/ # UserRepository, WorkspaceRepository, ProjectRepository, MembershipRepository, SummarizationTemplateRepository -│ │ ├── unit_of_work.py # UnitOfWork protocol (supports_* capability checks) -│ │ ├── async_context.py # Async context utilities -│ │ ├── diarization.py # DiarizationEngine protocol -│ │ ├── ner.py # NEREngine protocol -│ │ └── calendar.py # CalendarProvider protocol -│ ├── webhooks/ # WebhookEventType, WebhookConfig, WebhookDelivery, payloads -│ ├── triggers/ # Trigger, TriggerAction, TriggerSignal, TriggerProvider -│ ├── summarization/# SummarizationProvider protocol -│ ├── rules/ # Business rules registry, models, builtin rules -│ ├── settings/ # Domain settings base -│ ├── constants/ # Field definitions, placeholders -│ ├── utils/ # time.py (utc_now), validation.py -│ ├── errors.py # Domain-specific exceptions -│ └── value_objects.py -├── application/ # Use-cases/services -│ ├── services/ -│ │ ├── meeting/ # MeetingService (CRUD, segments, annotations, summaries, state) -│ │ ├── identity/ # IdentityService (context, workspace, defaults) -│ │ ├── calendar/ # CalendarService (connection, events, oauth, sync) -│ │ ├── summarization/ # SummarizationService, TemplateService, ConsentManager -│ │ ├── project_service/ # CRUD, members, roles, rules, active project -│ │ ├── recovery/ # RecoveryService (meeting, job, audio recovery) -│ │ ├── webhook_service.py -│ │ ├── ner_service.py -│ │ ├── export_service.py -│ │ ├── trigger_service.py -│ │ ├── retention_service.py -│ │ ├── auth_service.py -│ │ ├── auth_workflows.py -│ │ ├── auth_integration_manager.py -│ │ ├── auth_token_exchanger.py -│ │ ├── auth_types.py -│ │ ├── auth_constants.py -│ │ └── protocols.py -│ └── observability/ # Observability ports -├── infrastructure/ # Implementations -│ ├── audio/ # sounddevice capture, ring buffer, VU levels, playback, writer, reader -│ ├── asr/ # faster-whisper engine, VAD segmenter, streaming, DTOs -│ ├── auth/ # OIDC discovery, registry, presets -│ ├── diarization/ # Session, assigner, engine (streaming: diart, offline: pyannote) -│ ├── summarization/# CloudProvider, OllamaProvider, MockProvider, parsing, citation verifier, template renderer -│ ├── triggers/ # Calendar, audio_activity, foreground_app providers -│ ├── persistence/ # SQLAlchemy + asyncpg + pgvector -│ │ ├── database.py # create_async_engine, create_async_session_factory -│ │ ├── models/ # ORM models (core/, identity/, integrations/, entities/, observability/, organization/) -│ │ ├── repositories/ # Repository implementations -│ │ │ ├── meeting_repo.py -│ │ │ ├── segment_repo.py -│ │ │ ├── summary_repo.py -│ │ │ ├── annotation_repo.py -│ │ │ ├── entity_repo.py -│ │ │ ├── webhook_repo.py -│ │ │ ├── preferences_repo.py -│ │ │ ├── asset_repo.py -│ │ │ ├── summarization_template_repo.py -│ │ │ ├── diarization_job/ -│ │ │ ├── integration/ -│ │ │ ├── identity/ -│ │ │ ├── usage_event/ -│ │ │ └── _base/ # BaseRepository with _execute_scalar, _execute_scalars, _add_and_flush -│ │ ├── unit_of_work/ -│ │ ├── memory/ # In-memory implementations -│ │ └── migrations/ # Alembic migrations -│ ├── security/ # Keystore (keyring + AES-GCM), protocols, crypto/ -│ ├── crypto/ # Cryptographic utilities -│ ├── export/ # Markdown, HTML, PDF (WeasyPrint), formatting helpers -│ ├── webhooks/ # WebhookExecutor (delivery, signing, metrics) -│ ├── converters/ # ORM ↔ domain (orm, webhook, ner, calendar, integration, asr) -│ ├── calendar/ # Google/Outlook adapters, OAuth flow -│ ├── auth/ # OIDC registry, discovery, presets -│ ├── ner/ # spaCy NER engine -│ ├── observability/# OpenTelemetry tracing (otel.py), usage event tracking -│ ├── metrics/ # Metric collection utilities -│ ├── logging/ # Log buffer and utilities -│ └── platform/ # Platform-specific code -├── grpc/ # gRPC layer -│ ├── proto/ # noteflow.proto, generated *_pb2.py/*_pb2_grpc.py -│ ├── server/ # Bootstrap, lifecycle, setup, services, types -│ ├── service.py # NoteFlowServicer -│ ├── client.py # Python gRPC client wrapper -│ ├── meeting_store.py -│ ├── stream_state.py -│ ├── interceptors/ # Identity context propagation -│ ├── _mixins/ # Server-side gRPC mixins (see below) -│ └── _client_mixins/# Client-side gRPC mixins -├── cli/ # CLI tools -│ ├── __main__.py # CLI entry point -│ ├── retention.py # Retention management -│ ├── constants.py # CLI constants -│ └── models/ # Model commands (package) -│ ├── _download.py -│ ├── _parser.py -│ ├── _registry.py -│ ├── _status.py -│ └── _types.py -└── config/ # Pydantic settings (NOTEFLOW_ env vars) - ├── settings/ # _main.py, _features.py, _triggers.py, _calendar.py, _loaders.py - └── constants/ -``` - -### gRPC Server Mixins (`grpc/_mixins/`) - -``` -_mixins/ -├── streaming/ # ASR streaming (package) -│ ├── _mixin.py # Main StreamingMixin -│ ├── _session.py # Session management -│ ├── _asr.py # ASR processing -│ ├── _processing/ # Audio processing pipeline -│ │ ├── _audio_ops.py -│ │ ├── _chunk_tracking.py -│ │ ├── _congestion.py -│ │ ├── _constants.py -│ │ ├── _types.py -│ │ └── _vad_processing.py -│ ├── _partials.py # Partial transcript handling -│ ├── _cleanup.py # Resource cleanup -│ └── _types.py -├── diarization/ # Speaker diarization (package) -│ ├── _mixin.py # Main DiarizationMixin -│ ├── _jobs.py # Background job management -│ ├── _refinement.py# Offline refinement -│ ├── _streaming.py # Real-time diarization -│ ├── _speaker.py # Speaker assignment -│ ├── _status.py # Job status tracking -│ └── _types.py -├── summarization/ # Summary generation (package) -│ ├── _generation_mixin.py -│ ├── _templates_mixin.py -│ ├── _consent_mixin.py -│ ├── _template_crud.py -│ ├── _template_resolution.py -│ ├── _summary_generation.py -│ ├── _consent.py -│ └── _context_builders.py -├── meeting/ # Meeting lifecycle (package) -│ ├── meeting_mixin.py -│ ├── _project_scope.py -│ └── _stop_ops.py -├── project/ # Project management (package) -│ ├── _mixin.py -│ ├── _membership.py -│ └── _converters.py -├── oidc/ # OIDC authentication (package) -│ ├── oidc_mixin.py -│ └── _support.py -├── converters/ # Proto ↔ domain converters (package) -│ ├── _domain.py -│ ├── _external.py -│ ├── _timestamps.py -│ ├── _id_parsing.py -│ └── _oidc.py -├── errors/ # gRPC error helpers (package) -│ ├── _abort.py # abort_not_found, abort_invalid_argument -│ ├── _require.py # Requirement checks -│ ├── _fetch.py # Fetch with error handling -│ ├── _parse.py # Parsing helpers -│ └── _constants.py -├── servicer_core/ # Core servicer protocols -├── servicer_other/ # Additional servicer protocols -├── annotation.py # Segment annotations CRUD -├── export.py # Markdown/HTML/PDF export -├── entities.py # Named entity extraction -├── calendar.py # Calendar sync operations -├── webhooks.py # Webhook management -├── preferences.py # User preferences -├── observability.py # Usage tracking, metrics -├── identity.py # User/workspace identity -├── sync.py # State synchronization -├── diarization_job.py# Job status/management -├── protocols.py # ServicerHost protocol -├── _types.py -├── _audio_processing.py -├── _repository_protocols.py -└── _servicer_state.py -``` - -### gRPC Client Mixins (`grpc/_client_mixins/`) - -``` -_client_mixins/ -├── streaming.py # Client streaming operations -├── meeting.py # Meeting CRUD operations -├── diarization.py # Diarization requests -├── export.py # Export requests -├── annotation.py # Annotation operations -├── converters.py # Response converters -└── protocols.py # ClientHost protocol +├── src/noteflow/ # Python backend +│ ├── domain/ # Domain entities and ports +│ ├── application/ # Use cases and services +│ ├── infrastructure/ # External implementations +│ ├── grpc/ # gRPC server and client +│ ├── cli/ # Command-line tools +│ └── config/ # Configuration +├── client/ # Tauri + React desktop client +│ ├── src/ # React/TypeScript frontend +│ ├── src-tauri/ # Rust backend +│ └── e2e/ # End-to-end tests +├── tests/ # Python test suites +├── docs/ # Documentation +├── scripts/ # Development scripts +└── docker/ # Docker configurations ``` --- -## Client Architecture (Tauri + React) +## Key Cross-Cutting Concerns -``` -client/src/ -├── api/ # API layer -│ ├── tauri-adapter.ts # Main Tauri IPC adapter -│ ├── mock-adapter.ts # Mock adapter for testing -│ ├── cached-adapter.ts # Caching layer -│ ├── connection-state.ts # Connection state machine -│ ├── reconnection.ts # Auto-reconnection logic -│ ├── interface.ts # Adapter interface -│ ├── transcription-stream.ts -│ ├── types/ # Type definitions -│ │ ├── core.ts # Core types (Meeting, Segment, etc.) -│ │ ├── enums.ts # Enum definitions -│ │ ├── errors.ts # Error types -│ │ ├── projects.ts # Project types -│ │ ├── diagnostics.ts -│ │ ├── requests.ts -│ │ ├── features/ # Feature-specific types -│ │ │ ├── webhooks.ts -│ │ │ ├── calendar.ts -│ │ │ ├── ner.ts -│ │ │ ├── sync.ts -│ │ │ ├── identity.ts -│ │ │ ├── oidc.ts -│ │ │ └── observability.ts -│ │ └── requests/ # Request types by domain -│ └── cached/ # Cached adapter implementations -│ ├── base.ts -│ ├── meetings.ts -│ ├── projects.ts -│ ├── diarization.ts -│ ├── annotations.ts -│ ├── templates.ts -│ ├── webhooks.ts -│ ├── calendar.ts -│ ├── entities.ts -│ ├── preferences.ts -│ ├── observability.ts -│ ├── triggers.ts -│ ├── audio.ts -│ ├── playback.ts -│ ├── apps.ts -│ └── readonly.ts -├── hooks/ # Custom React hooks -│ ├── use-diarization.ts -│ ├── use-cloud-consent.ts -│ ├── use-webhooks.ts -│ ├── use-oauth-flow.ts -│ ├── use-calendar-sync.ts -│ ├── use-entity-extraction.ts -│ ├── use-audio-devices.ts -│ ├── use-project.ts -│ ├── use-project-members.ts -│ ├── use-oidc-providers.ts -│ ├── use-auth-flow.ts -│ ├── use-integration-sync.ts -│ ├── use-integration-validation.ts -│ ├── use-secure-integration-secrets.ts -│ ├── use-guarded-mutation.ts -│ ├── use-panel-preferences.ts -│ ├── use-preferences-sync.ts -│ ├── use-meeting-reminders.ts -│ ├── use-recording-app-policy.ts -│ ├── use-post-processing.ts -│ ├── use-toast.ts -│ ├── use-mobile.tsx -│ └── post-processing/ -├── contexts/ # React contexts -│ ├── connection-context.tsx # gRPC connection context -│ ├── connection-state.ts -│ ├── workspace-context.tsx # Workspace context -│ ├── workspace-state.ts -│ ├── project-context.tsx # Project context -│ └── project-state.ts -├── components/ # React components -│ ├── ui/ # Reusable UI components (shadcn/ui) -│ ├── recording/ # Recording-specific components -│ ├── settings/ # Settings panel components -│ ├── analytics/ # Analytics visualizations -│ ├── projects/ # Project components -│ ├── icons/ # Icon components -│ └── ... # Top-level components -├── pages/ # Route pages -│ ├── Home.tsx -│ ├── Meetings.tsx -│ ├── MeetingDetail.tsx -│ ├── Recording.tsx -│ ├── Projects.tsx -│ ├── ProjectSettings.tsx -│ ├── Settings.tsx -│ ├── settings/ # Settings sub-pages -│ ├── People.tsx -│ ├── Tasks.tsx -│ ├── Analytics.tsx -│ ├── Index.tsx -│ └── NotFound.tsx -├── lib/ # Utilities -│ ├── config/ # Configuration (server, defaults, app-config, provider-endpoints) -│ ├── cache/ # Client-side caching (meeting-cache) -│ ├── format.ts -│ ├── utils.ts -│ ├── time.ts -│ ├── crypto.ts -│ ├── cva.ts -│ ├── styles.ts -│ ├── tauri-events.ts -│ ├── preferences.ts -│ ├── preferences-sync.ts -│ ├── entity-store.ts -│ ├── speaker-utils.ts -│ ├── ai-providers.ts -│ ├── ai-models.ts -│ ├── integration-utils.ts -│ ├── default-integrations.ts -│ ├── status-constants.ts -│ ├── timing-constants.ts -│ ├── object-utils.ts -│ ├── error-reporting.ts -│ ├── client-logs.ts -│ ├── client-log-events.ts -│ ├── log-groups.ts -│ ├── log-converters.ts -│ ├── log-messages.ts -│ ├── log-summarizer.ts -│ └── log-group-summarizer.ts -├── types/ # Shared TypeScript types -└── test/ # Test utilities -``` +### gRPC Schema Synchronization -### Rust/Tauri Backend (`client/src-tauri/src/`) +When modifying `src/noteflow/grpc/proto/noteflow.proto`: -``` -src-tauri/src/ -├── commands/ # Tauri IPC command handlers -│ ├── recording/ # capture.rs, device.rs, audio.rs, app_policy.rs -│ ├── playback/ # audio.rs, events.rs, tick.rs -│ ├── triggers/ # audio.rs, polling.rs -│ ├── meeting.rs -│ ├── diarization.rs -│ ├── annotation.rs -│ ├── export.rs -│ ├── summary.rs -│ ├── entities.rs -│ ├── calendar.rs -│ ├── webhooks.rs -│ ├── preferences.rs -│ ├── observability.rs -│ ├── sync.rs -│ ├── projects.rs -│ ├── identity.rs -│ ├── oidc.rs -│ ├── connection.rs -│ ├── audio.rs -│ ├── audio_testing.rs -│ ├── apps.rs -│ ├── apps_platform.rs -│ ├── diagnostics.rs -│ ├── shell.rs -│ └── testing.rs -├── grpc/ # gRPC client -│ ├── client/ # Client implementations by domain -│ │ ├── core.rs -│ │ ├── meetings.rs -│ │ ├── annotations.rs -│ │ ├── diarization.rs -│ │ ├── identity.rs -│ │ ├── projects.rs -│ │ ├── preferences.rs -│ │ ├── calendar.rs -│ │ ├── webhooks.rs -│ │ ├── observability.rs -│ │ ├── oidc.rs -│ │ ├── sync.rs -│ │ └── converters.rs -│ ├── types/ # Rust type definitions -│ │ ├── core.rs -│ │ ├── enums.rs -│ │ ├── identity.rs -│ │ ├── projects.rs -│ │ ├── preferences.rs -│ │ ├── calendar.rs -│ │ ├── webhooks.rs -│ │ ├── observability.rs -│ │ ├── oidc.rs -│ │ ├── sync.rs -│ │ └── results.rs -│ ├── streaming/ # Streaming converters -│ └── noteflow.rs # Generated protobuf types -├── state/ # Runtime state management -│ ├── app_state.rs -│ ├── preferences.rs -│ ├── playback.rs -│ └── types.rs -├── audio/ # Audio capture and playback -├── cache/ # Memory caching -├── crypto/ # Cryptographic operations -├── events/ # Tauri event emission -├── triggers/ # Trigger detection -├── error/ # Error types -├── identity/ # Identity management -├── config.rs # Configuration -├── constants.rs # Constants -├── helpers.rs # Helper functions -├── oauth_loopback.rs # OAuth callback server -├── main.rs # Application entry point -└── lib.rs # Library exports -``` +1. **Python**: Regenerate stubs and run `python scripts/patch_grpc_stubs.py` +2. **Rust**: Rebuild Tauri (`cd client && npm run tauri:build`) +3. **TypeScript**: Update types in `client/src/api/types/` + +### Code Quality Standards + +- **Python**: Basedpyright strict mode, Ruff linting, 100-char line limit +- **TypeScript**: Biome linting, strict mode, single quotes +- **Rust**: rustfmt formatting, clippy linting + +### Testing Philosophy + +- **Python**: pytest with async support, testcontainers for integration +- **TypeScript**: Vitest unit tests, Playwright E2E +- **Rust**: cargo test with integration test support --- -## Database - -PostgreSQL with pgvector extension. Async SQLAlchemy with asyncpg driver. - -```bash -# Alembic migrations -alembic upgrade head -alembic revision --autogenerate -m "description" -``` - -Connection via `NOTEFLOW_DATABASE_URL` env var or settings. - -### ORM Models (`persistence/models/`) - -| Directory | Models | -|-----------|--------| -| `core/` | MeetingModel, SegmentModel, SummaryModel, AnnotationModel, DiarizationJobModel | -| `identity/` | UserModel, WorkspaceModel, WorkspaceMembershipModel, ProjectModel, ProjectMembershipModel, SettingsModel | -| `integrations/` | IntegrationModel, IntegrationSecretModel, CalendarEventModel, MeetingCalendarLinkModel, WebhookConfigModel, WebhookDeliveryModel | -| `entities/` | NamedEntityModel, SpeakerModel | -| `observability/` | UsageEventModel | -| `organization/` | SummarizationTemplateModel, TaskModel, TagModel | - ---- - -## Testing Conventions - -- Test files: `test_*.py`, functions: `test_*` -- Markers: `@pytest.mark.slow` (model loading), `@pytest.mark.integration` (external services) -- Integration tests use testcontainers for PostgreSQL -- Asyncio auto-mode enabled -- React unit tests use Vitest; e2e tests use Playwright in `client/e2e/` - -### Test Quality Gates (`tests/quality/`) - -**After any non-trivial changes**, run: - -```bash -pytest tests/quality/ -``` - -This suite enforces: - -| Check | Description | -|-------|-------------| -| `test_test_smells.py` | No assertion roulette, no conditional test logic, no loops in tests | -| `test_magic_values.py` | No magic numbers in assignments | -| `test_code_smells.py` | Code quality checks | -| `test_duplicate_code.py` | No duplicate code patterns | -| `test_stale_code.py` | No stale/dead code | -| `test_decentralized_helpers.py` | Helpers consolidated properly | -| `test_unnecessary_wrappers.py` | No unnecessary wrapper functions | -| `test_baseline_self.py` | Baseline validation self-checks | - -### Global Fixtures (`tests/conftest.py`) - -**Do not redefine these fixtures:** - -| Fixture | Description | -|---------|-------------| -| `reset_context_vars` | Reset logging context variables | -| `mock_uow` | Mock Unit of Work | -| `crypto` | Crypto utilities | -| `meetings_dir` | Temporary meetings directory | -| `webhook_config` | Single-event webhook config | -| `webhook_config_all_events` | All-events webhook config | -| `sample_datetime` | Sample datetime | -| `calendar_settings` | Calendar settings | -| `meeting_id` | Sample meeting ID | -| `sample_meeting` | Sample Meeting entity | -| `recording_meeting` | Recording-state Meeting | -| `sample_rate` | Audio sample rate | -| `mock_grpc_context` | Mock gRPC context | -| `mockasr_engine` | Mock ASR engine | -| `mock_optional_extras` | Mock optional extras | -| `mock_oauth_manager` | Mock OAuth manager | -| `memory_servicer` | In-memory servicer | -| `approx_float` | Approximate float comparison | -| `approx_sequence` | Approximate sequence comparison | - ---- - -## Code Reuse (CRITICAL) - -**BEFORE writing ANY new code, you MUST search for existing implementations.** - -This is not optional. Redundant code creates maintenance burden, inconsistency, and bugs. - -### Mandatory Search Process - -1. **Search for existing functions** that do what you need: - ```bash - # Use Serena's symbolic tools first - find_symbol with substring_matching=true - search_for_pattern for broader searches - ``` - -2. **Check related modules** - if you need audio device config, check `device.rs`; if you need preferences, check `preferences.ts` - -3. **Look at imports** in similar files - they reveal available utilities - -4. **Only create new code if:** - - No existing implementation exists - - Existing code cannot be reasonably extended - - You have explicit approval for new abstractions - -### Anti-Patterns (FORBIDDEN) - -| Anti-Pattern | Correct Approach | -|--------------|------------------| -| Creating wrapper functions for existing utilities | Use the existing function directly | -| Duplicating validation logic | Find and reuse existing validators | -| Writing new helpers without searching | Search first, ask if unsure | -| "It's faster to write new code" | Technical debt is never faster | - -### Examples - -**BAD:** Creating `query_capture_config()` when `resolve_input_device()` + `select_input_config()` already exist - -**GOOD:** Importing and using existing functions directly: -```rust -use device::{resolve_input_device, select_input_config}; -// ... use them directly -``` - -**BAD:** Writing a new date formatting function - -**GOOD:** Searching for existing formatters in `lib/format.ts` or `utils/` - ---- - -## Code Style - -### Python - -- Python 3.12+, 100-char line length -- 4-space indentation -- Naming: `snake_case` modules/functions, `PascalCase` classes, `UPPER_SNAKE_CASE` constants -- Strict basedpyright (0 errors, 0 warnings, 0 notes required) -- Ruff for linting (E, W, F, I, B, C4, UP, SIM, RUF) -- Module soft limit 500 LoC, hard limit 750 LoC -- Generated `*_pb2.py`, `*_pb2_grpc.py` excluded from lint - -### TypeScript - -- Biome for linting and formatting -- Single quotes, 100 char width -- Strict TypeScript (noEmit checks) - -### Rust - -- `rustfmt` for formatting -- `clippy` for linting (warnings treated as errors) - -### Client Logging (CRITICAL) - -**NEVER use `console.log`, `console.error`, `console.warn`, or `console.debug` in client TypeScript code.** - -Always use the `clientlog` system via `client/src/lib/debug.ts` or `client/src/lib/client-logs.ts`: - -```typescript -// For debug logging (controlled by DEBUG flag) -import { debug } from '@/lib/debug'; -const log = debug('MyComponent'); -log('Something happened', { detail: 'value' }); - -// For error logging (always outputs) -import { errorLog } from '@/lib/debug'; -const logError = errorLog('MyComponent'); -logError('Something failed', error); - -// For direct clientlog access -import { addClientLog } from '@/lib/client-logs'; -addClientLog({ - level: 'info', - source: 'app', - message: 'Event occurred', - details: 'Additional context', -}); -``` - -**Why:** -- `clientlog` persists logs to localStorage for later viewing in Analytics -- Logs are structured with level, source, timestamp, and metadata -- Debug logs can be toggled at runtime via `DEBUG=true` in localStorage -- Console logging is ephemeral and not accessible to users - ---- - -## Type Safety (Zero Tolerance) - -### Forbidden Patterns (Python) - -| Pattern | Why Blocked | Alternative | -|---------|-------------|-------------| -| `# type: ignore` | Bypasses type safety | Fix the actual type error | -| `# pyright: ignore` | Bypasses type safety | Fix the actual type error | -| `Any` type annotations | Creates type safety holes | Use `Protocol`, `TypeVar`, `TypedDict`, or specific types | -| Magic numbers | Hidden intent | Define `typing.Final` constants | -| Loops in tests | Non-deterministic | Use `@pytest.mark.parametrize` | -| Conditionals in tests | Non-deterministic | Use `@pytest.mark.parametrize` | -| Multiple assertions without messages | Hard to debug | Add assertion messages | - -### Type Resolution Hierarchy - -When facing dynamic types: - -1. **`Protocol`** — For duck typing (structural subtyping) -2. **`TypeVar`** — For generics -3. **`TypedDict`** — For structured dictionaries -4. **`cast()`** — Last resort (with comment explaining why) - -### Validation Requirements - -After any code changes: - -```bash -source .venv/bin/activate && basedpyright src/noteflow/ -``` - -**Expected output:** `0 errors, 0 warnings, 0 notes` - ---- - -## Automated Enforcement (Hookify Rules) - -### Protected Files (Require Explicit Permission) - -| File/Directory | What's Blocked | -|----------------|----------------| -| `Makefile` | All modifications | -| `tests/quality/` (except `baselines.json`) | All modifications | -| `client/src/test/code-quality.test.ts` | All modifications to allowlists/thresholds | -| `pyproject.toml`, `ruff.toml`, `pyrightconfig.json` | All edits | -| `biome.json`, `tsconfig.json`, `.eslintrc*` | All edits | -| `.rustfmt.toml`, `.clippy.toml` | All edits | - -### Quality Gate Requirement - -Before completing any code changes: - -```bash -make quality -``` - -All quality checks must pass. - -### Policy: No Ignoring Pre-existing Issues - -If you encounter lint errors, type errors, or test failures—**even if they existed before your changes**—you must either: - -1. Fix immediately (for simple issues) -2. Add to todo list (for complex issues) -3. Launch a subagent to fix (for parallelizable work) - -### Policy: Never Modify Quality Test Allowlists - -**STRICTLY FORBIDDEN** without explicit user permission: - -1. Adding entries to allowlists/whitelists in quality tests (e.g., `allowedNumbers`, `ALLOWED_LONG_FILES`, `allowedStorageFiles`) -2. Increasing thresholds (e.g., `toBeLessThanOrEqual(0)` → `toBeLessThanOrEqual(5)`) -3. Adding exclusion patterns to skip files from quality checks -4. Modifying filter functions to bypass detection (e.g., `isNotMagicNumber`) -5. **Reading or accessing quality test files to check allowlist contents** — there is never a valid reason to inspect what's in an allowlist; always fix the actual code instead - -**When quality tests fail, the correct approach is:** - -1. **Fix the actual code** that triggers the violation (refactor, extract, consolidate) -2. If the detection is a false positive (e.g., sprint references in comments), **improve the filter logic** to correctly exclude false positives while still catching real issues -3. **Never** add arbitrary values to allowlists just to make tests pass -4. **Never** read allowlist files to see "what's allowed" — this creates temptation to add entries - -**Example of WRONG approach:** -```typescript -// BAD: Adding sprint numbers to allowlist to avoid fixing code -const allowedNumbers = ['100', '200', '001', '002', '003']; // ❌ - -// BAD: Reading allowlist to check what files are exempt -Grep("ALLOWED_LONG_FILES", "code-quality.test.ts") // ❌ -``` - -**Example of CORRECT approach:** -```typescript -// GOOD: Improve filter to detect sprint references in comments -if (/(?:GAP|Sprint)[- ]?\d+/i.test(content)) return true; // ✓ - -// GOOD: Refactor long file by extracting types/helpers to separate modules -// file.ts (520 lines) → file.ts (480 lines) + file.types.ts (40 lines) -``` - ---- - -## Proto/gRPC - -Proto definitions: `src/noteflow/grpc/proto/noteflow.proto` - -Regenerate after proto changes: - -```bash -python -m grpc_tools.protoc -I src/noteflow/grpc/proto \ - --python_out=src/noteflow/grpc/proto \ - --grpc_python_out=src/noteflow/grpc/proto \ - src/noteflow/grpc/proto/noteflow.proto -``` - -Then run stub patching: - -```bash -python scripts/patch_grpc_stubs.py -``` - -### Sync Points (High Risk of Breakage) - -When changing proto: - -1. **Python stubs** — Regenerate `*_pb2.py`, `*_pb2_grpc.py` -2. **Server mixins** — Update `src/noteflow/grpc/_mixins/` -3. **Python client** — Update `src/noteflow/grpc/client.py` -4. **Rust types** — Generated by `client/src-tauri/build.rs` -5. **Rust commands** — Update `client/src-tauri/src/commands/` -6. **TypeScript types** — Update `client/src/api/types/` -7. **TypeScript adapter** — Update `client/src/api/tauri-adapter.ts` - ---- - -## Key Subsystems - -### Speaker Diarization - -- **Streaming**: diart for real-time speaker detection -- **Offline**: pyannote.audio for post-meeting refinement -- **gRPC**: `RefineSpeakerDiarization`, `GetDiarizationJobStatus`, `RenameSpeaker` - -### Summarization - -- **Providers**: CloudProvider (Anthropic/OpenAI), OllamaProvider (local), MockProvider (testing) -- **Templates**: Configurable tone, format, verbosity -- **Citation verification**: Links summary claims to transcript evidence -- **Consent**: Cloud providers require explicit user consent - -### Export - -- **Formats**: Markdown, HTML, PDF (via WeasyPrint) -- **Content**: Transcript with timestamps, speaker labels, summary - -### Named Entity Recognition (NER) - -- **Engine**: spaCy with transformer models -- **Categories**: person, company, product, technical, acronym, location, date, other -- **Segment tracking**: Entities link to source `segment_ids` - -### Trigger Detection - -- **Signals**: Calendar proximity, audio activity, foreground app -- **Actions**: IGNORE, NOTIFY, AUTO_START - -### Webhooks - -- **Events**: `meeting.completed`, `summary.generated`, `recording.started`, `recording.stopped` -- **Delivery**: Exponential backoff retries -- **Security**: HMAC-SHA256 signing - -### Authentication - -- **OIDC**: OpenID Connect with discovery -- **Providers**: Configurable via OIDC registry - ---- - -## Feature Flags - -| Flag | Default | Controls | -|------|---------|----------| -| `NOTEFLOW_FEATURE_TEMPLATES_ENABLED` | `true` | AI summarization templates | -| `NOTEFLOW_FEATURE_PDF_EXPORT_ENABLED` | `true` | PDF export format | -| `NOTEFLOW_FEATURE_NER_ENABLED` | `false` | Named entity extraction | -| `NOTEFLOW_FEATURE_CALENDAR_ENABLED` | `false` | Calendar sync | -| `NOTEFLOW_FEATURE_WEBHOOKS_ENABLED` | `true` | Webhook notifications | - -Access via `get_feature_flags().` or `get_settings().feature_flags.`. - ---- - -## Common Pitfalls Checklist - -### When Adding New Features - -- [ ] Update proto schema first (if gRPC involved) -- [ ] Regenerate Python stubs -- [ ] Run `scripts/patch_grpc_stubs.py` -- [ ] Implement server mixin -- [ ] Update Python client wrapper -- [ ] Update Rust commands -- [ ] Update TypeScript adapter -- [ ] Update TypeScript types -- [ ] Add tests (both backend and client) -- [ ] Run `make quality` - -### When Changing Database Schema - -- [ ] Update ORM models in `persistence/models/` -- [ ] Create Alembic migration -- [ ] Update repository implementation -- [ ] Update UnitOfWork if needed -- [ ] Update converters in `infrastructure/converters/` - -### When Modifying Existing Code - -- [ ] Search for all usages first -- [ ] Update all call sites -- [ ] Run `make quality` -- [ ] Run relevant tests +## Development Workflow + +1. **Setup**: Install Python and client dependencies +2. **Database**: Start PostgreSQL with `docker compose up -d postgres` +3. **Backend**: Run `python -m noteflow.grpc.server` +4. **Frontend**: Run `cd client && npm run tauri dev` +5. **Quality**: Run `make quality` before committing +6. **Tests**: Run full test suite with `pytest && cd client && npm run test:all` --- @@ -953,72 +330,3 @@ Access via `get_feature_flags().` or `get_settings().feature_flags.500 lines flagged) -### Logging (CRITICAL) +### Client Logging (CRITICAL) **NEVER use `console.log`, `console.error`, `console.warn`, or `console.debug` directly.** diff --git a/client/src/CLAUDE.md b/client/src/CLAUDE.md new file mode 100644 index 0000000..f415560 --- /dev/null +++ b/client/src/CLAUDE.md @@ -0,0 +1,338 @@ +# TypeScript & JavaScript Security Rules + +Comprehensive security guidelines for TypeScript and JavaScript development in NoteFlow. + +## TypeScript Security + +### Type Safety + +**Strict Configuration (warning, CWE-704)** +```json +{ + "compilerOptions": { + "strict": true, + "noImplicitAny": true, + "strictNullChecks": true, + "noUncheckedIndexedAccess": true, + "exactOptionalPropertyTypes": true + } +} +``` + +**Runtime Validation (strict, CWE-20, OWASP A03:2025)** +```typescript +import { z } from 'zod'; + +const UserSchema = z.object({ + id: z.number(), + email: z.string().email(), + role: z.enum(['user', 'admin']), +}); + +type User = z.infer; + +async function fetchUser(id: number): Promise { + const response = await fetch(`/api/users/${id}`); + const data = await response.json(); + return UserSchema.parse(data); // Runtime validation +} +``` + +**Avoid Type Assertions (strict, CWE-704, CWE-20)** +```typescript +// Do: Use type guards +function isUser(obj: unknown): obj is User { + return typeof obj === 'object' && obj !== null && 'id' in obj; +} + +// Don't: Bypass type safety +const data = JSON.parse(text) as unknown as SecretData; +``` + +### API Security + +**Separate Internal/API Types (warning, CWE-200, OWASP A01:2025)** +```typescript +interface UserInternal { + id: number; + email: string; + passwordHash: string; // Internal only + createdAt: Date; +} + +interface UserResponse { + id: number; + email: string; + createdAt: string; // ISO string for API +} + +function toUserResponse(user: UserInternal): UserResponse { + return { + id: user.id, + email: user.email, + createdAt: user.createdAt.toISOString(), + }; +} +``` + +**Branded Types for Sensitive Data (advisory, CWE-704)** +```typescript +type UserId = string & { readonly brand: unique symbol }; +type PostId = string & { readonly brand: unique symbol }; + +function getUser(id: UserId): User { } // Type-safe ID handling +``` + +### Null & Enum Safety + +**Explicit Null Handling (warning, CWE-476)** +```typescript +function getUserEmail(userId: string): string | null { + const user = users.get(userId); + if (!user) return null; + return user.email; +} + +// Optional chaining with nullish coalescing +const email = user?.email ?? 'default@example.com'; +``` + +**String Enums for External Data (warning, CWE-20)** +```typescript +enum UserRole { + Admin = 'admin', + User = 'user', + Guest = 'guest', +} + +// Or const objects +const UserRole = { + Admin: 'admin', + User: 'user', + Guest: 'guest', +} as const; + +type UserRole = typeof UserRole[keyof typeof UserRole]; +``` + +## JavaScript Security + +### Code Execution Prevention + +**No eval() with User Input (strict, CWE-94, CWE-95, OWASP A03:2025)** +```javascript +// Do: Use JSON.parse for data +const data = JSON.parse(userInput); + +// Do: Use Map for dynamic dispatch +const handlers = new Map([ + ['action1', handleAction1], + ['action2', handleAction2] +]); + +// Don't: Execute arbitrary code +eval(userInput); +new Function(userInput)(); +setTimeout(userCode, 1000); +``` + +**Prevent Prototype Pollution (strict, CWE-1321)** +```javascript +// Do: Use Object.create(null) for dictionaries +const safeDict = Object.create(null); + +// Do: Validate keys before assignment +function safeSet(obj, key, value) { + if (key === '__proto__' || key === 'constructor' || key === 'prototype') { + throw new Error('Invalid key'); + } + obj[key] = value; +} + +// Don't: Direct assignment with user keys +obj[userKey] = userValue; // Can pollute prototype +``` + +### DOM Security + +**Sanitize HTML Before Insertion (strict, CWE-79, OWASP A03:2025)** +```javascript +import DOMPurify from 'dompurify'; + +// Do: Sanitize HTML +const clean = DOMPurify.sanitize(userHtml); +element.innerHTML = clean; + +// Do: Use textContent for plain text +element.textContent = userInput; + +// Don't: Direct HTML insertion +element.innerHTML = userInput; // XSS vulnerability +document.write(userInput); // XSS via document.write +``` + +**Validate URLs Before Use (strict, CWE-601, CWE-79)** +```javascript +function isValidUrl(urlString) { + try { + const url = new URL(urlString); + return ['http:', 'https:'].includes(url.protocol); + } catch { + return false; + } +} + +// Safe redirect +if (isValidUrl(redirectUrl) && isSameDomain(redirectUrl)) { + window.location.href = redirectUrl; +} + +// Don't: Use unvalidated URLs +element.href = userUrl; // javascript: URLs execute code +``` + +### Node.js Security + +**Prevent Command Injection (strict, CWE-78, OWASP A03:2025)** +```javascript +const { execFile } = require('child_process'); + +// Do: Use execFile with argument array +execFile('grep', [pattern, filename], (error, stdout) => { + console.log(stdout); +}); + +// Don't: Shell string interpolation +exec(`grep ${userPattern} ${userFile}`); // Command injection +exec('ls ' + userInput); // Shell interpretation +``` + +**Validate File Paths (strict, CWE-22, OWASP A01:2025)** +```javascript +const path = require('path'); +const SAFE_DIR = '/app/uploads'; + +function safeReadFile(filename) { + const resolved = path.resolve(SAFE_DIR, filename); + + // Ensure path is within safe directory + if (!resolved.startsWith(SAFE_DIR + path.sep)) { + throw new Error('Path traversal detected'); + } + + return fs.readFileSync(resolved); +} + +// Don't: Unvalidated path concatenation +fs.readFileSync(`./uploads/${userFilename}`); // Path traversal +``` + +**Secure Dependencies (warning, CWE-1104, OWASP A06:2025)** +```bash +npm audit # Audit dependencies +npm ci # Use lockfile +npm outdated # Check for updates +``` + +```json +{ + "dependencies": { + "express": "4.18.2" // Pin exact versions + } +} +``` + +### Cryptography + +**Use Crypto Module Correctly (strict, CWE-330, CWE-328)** +```javascript +const crypto = require('crypto'); + +// Do: Secure random token +const token = crypto.randomBytes(32).toString('hex'); + +// Do: Secure UUID +const { randomUUID } = require('crypto'); +const id = randomUUID(); + +// Do: Password hashing +const bcrypt = require('bcrypt'); +const hash = await bcrypt.hash(password, 12); + +// Don't: Predictable randomness +const token = Math.random().toString(36); // Predictable + +// Don't: Weak password hashing +const hash = crypto.createHash('md5').update(password).digest('hex'); +``` + +### HTTP Security + +**Set Security Headers (warning, OWASP A05:2025)** +```javascript +const helmet = require('helmet'); +const app = express(); + +app.use(helmet()); // Comprehensive security headers + +// Or set individually +app.use((req, res, next) => { + res.setHeader('X-Content-Type-Options', 'nosniff'); + res.setHeader('X-Frame-Options', 'DENY'); + res.setHeader('Content-Security-Policy', "default-src 'self'"); + next(); +}); +``` + +**Configure CORS Properly (strict, CWE-942, OWASP A05:2025)** +```javascript +const cors = require('cors'); + +// Do: Specific origins only +app.use(cors({ + origin: ['https://myapp.com', 'https://admin.myapp.com'], + methods: ['GET', 'POST'], + credentials: true +})); + +// Don't: Permissive CORS +app.use(cors({ origin: '*', credentials: true })); // Vulnerable +app.use(cors({ origin: true })); // Reflects any origin +``` + +## Quick Reference + +| Rule | Level | CWE | OWASP | +|------|-------|-----|-------| +| **TypeScript** | +| Strict tsconfig | warning | CWE-704 | - | +| Runtime validation | strict | CWE-20 | A03:2025 | +| Avoid type assertions | strict | CWE-704 | - | +| Separate API types | warning | CWE-200 | A01:2025 | +| Branded types | advisory | CWE-704 | - | +| Null safety | warning | CWE-476 | - | +| String enums | warning | CWE-20 | - | +| **JavaScript** | +| No eval() | strict | CWE-94,95 | A03:2025 | +| Prototype pollution | strict | CWE-1321 | - | +| Sanitize HTML | strict | CWE-79 | A03:2025 | +| Validate URLs | strict | CWE-601 | - | +| Command injection | strict | CWE-78 | A03:2025 | +| Path traversal | strict | CWE-22 | A01:2025 | +| Secure dependencies | warning | CWE-1104 | A06:2025 | +| Crypto randomness | strict | CWE-330 | - | +| Security headers | warning | - | A05:2025 | +| CORS configuration | strict | CWE-942 | A05:2025 | + +## Key Principles + +1. **Never trust external data** - Always validate at runtime +2. **Type safety != runtime safety** - TypeScript types are erased +3. **Use secure defaults** - Explicitly configure security settings +4. **Principle of least privilege** - Minimize permissions and exposure +5. **Defense in depth** - Apply multiple security layers + +## Version History + +- **v2.0.0** - Combined TypeScript/JavaScript rules, compacted format +- **v1.0.0** - Initial separate TypeScript and JavaScript security rules \ No newline at end of file diff --git a/client/src/api/reconnection.ts b/client/src/api/reconnection.ts index e78042b..6182fa9 100644 --- a/client/src/api/reconnection.ts +++ b/client/src/api/reconnection.ts @@ -69,7 +69,9 @@ async function syncStateAfterReconnect(): Promise { try { const serverInfo = await getAPI().getServerInfo(); // Check if sync was superseded - if (syncGeneration !== currentGeneration) return; + if (syncGeneration !== currentGeneration) { + return; + } if (typeof serverInfo.state_version === 'number') { meetingCache.updateServerStateVersion(serverInfo.state_version); } @@ -84,13 +86,17 @@ async function syncStateAfterReconnect(): Promise { } // Check if sync was superseded - if (syncGeneration !== currentGeneration) return; + if (syncGeneration !== currentGeneration) { + return; + } // 3. Revalidate cached integrations against server try { await preferences.revalidateIntegrations(); // Check if sync was superseded - if (syncGeneration !== currentGeneration) return; + if (syncGeneration !== currentGeneration) { + return; + } } catch (error) { addClientLog({ level: 'warning', @@ -104,7 +110,9 @@ async function syncStateAfterReconnect(): Promise { // 4. Execute all registered reconnection callbacks for (const callback of reconnectionCallbacks) { // Check if sync was superseded before each callback - if (syncGeneration !== currentGeneration) return; + if (syncGeneration !== currentGeneration) { + return; + } try { await callback(); } catch (error) { diff --git a/client/src/components/entity-management-panel.tsx b/client/src/components/entity-management-panel.tsx index aac4ab9..3d96463 100644 --- a/client/src/components/entity-management-panel.tsx +++ b/client/src/components/entity-management-panel.tsx @@ -287,11 +287,15 @@ export function EntityManagementPanel({ meetingId }: EntityManagementPanelProps) }); } // Check mounted state before updating UI - if (!isMountedRef.current) return; + if (!isMountedRef.current) { + return; + } toast({ title: 'Entity updated', description: `"${data.text}" has been updated.` }); setEditingEntity(null); } catch (error) { - if (!isMountedRef.current) return; + if (!isMountedRef.current) { + return; + } toastError({ title: 'Update failed', error, @@ -316,11 +320,15 @@ export function EntityManagementPanel({ meetingId }: EntityManagementPanelProps) await deleteEntityWithPersist(meetingId, deletingEntity.id); } // Check mounted state before updating UI - if (!isMountedRef.current) return; + if (!isMountedRef.current) { + return; + } toast({ title: 'Entity deleted', description: `"${deletingEntity.text}" has been removed.` }); setDeletingEntity(null); } catch (error) { - if (!isMountedRef.current) return; + if (!isMountedRef.current) { + return; + } toastError({ title: 'Delete failed', error, diff --git a/client/src/components/settings/advanced-local-ai-settings/_constants.ts b/client/src/components/settings/advanced-local-ai-settings/_constants.ts index 8f1a20d..8b5f4b6 100644 --- a/client/src/components/settings/advanced-local-ai-settings/_constants.ts +++ b/client/src/components/settings/advanced-local-ai-settings/_constants.ts @@ -341,11 +341,9 @@ export function getSuggestedComputeType( if (types.includes('int8')) { return 'int8'; } - } else if (device === 'cpu') { - if (types.includes('int8')) { - return 'int8'; - } - } + } else if (device === 'cpu' && types.includes('int8')) { + return 'int8'; + } return types[0] ?? null; } @@ -361,11 +359,9 @@ export function getSuggestedComputeType( if (types.includes('int8')) { return 'int8'; } - } else if (device === 'cpu') { - if (types.includes('int8')) { - return 'int8'; - } - } + } else if (device === 'cpu' && types.includes('int8')) { + return 'int8'; + } return types[0] ?? null; } diff --git a/client/src/components/settings/advanced-local-ai-settings/model-auth-section.tsx b/client/src/components/settings/advanced-local-ai-settings/model-auth-section.tsx index 50996f1..727617b 100644 --- a/client/src/components/settings/advanced-local-ai-settings/model-auth-section.tsx +++ b/client/src/components/settings/advanced-local-ai-settings/model-auth-section.tsx @@ -46,7 +46,9 @@ export function ModelAuthSection({ const [showToken, setShowToken] = useState(false); const handleSaveToken = async () => { - if (!tokenInput.trim()) return; + if (!tokenInput.trim()) { + return; + } const success = await setToken(tokenInput.trim(), true); if (success) { setTokenInput(''); diff --git a/client/src/components/settings/ai-config-section.tsx b/client/src/components/settings/ai-config-section.tsx index 9789db6..6c41b27 100644 --- a/client/src/components/settings/ai-config-section.tsx +++ b/client/src/components/settings/ai-config-section.tsx @@ -294,9 +294,7 @@ export function AIConfigSection() { }); const sourceLabel = result.source === 'cache' ? 'Loaded from cache' : 'Loaded from API'; const forceLabel = forceRefresh ? ' (forced refresh)' : ''; - const description = result.error - ? result.error - : `${sourceLabel}${forceLabel} • ${nextModels.length} models`; + const description = result.error || `${sourceLabel}${forceLabel} • ${nextModels.length} models`; toast({ title: result.stale ? 'Models loaded (stale cache)' : 'Models loaded', description, diff --git a/client/src/hooks/use-auth-flow.ts b/client/src/hooks/use-auth-flow.ts index 1b5ca90..07fd131 100644 --- a/client/src/hooks/use-auth-flow.ts +++ b/client/src/hooks/use-auth-flow.ts @@ -98,11 +98,15 @@ export function useAuthFlow(): UseAuthFlowReturn { try { const response = await api.completeAuthLogin(provider, params.code, params.state); - if (unmounted) break; // Abort if unmounted during async operation + if (unmounted) { + break; + } // Abort if unmounted during async operation if (response.success) { const userInfo = await api.getCurrentUser(); - if (unmounted) break; // Abort if unmounted during async operation + if (unmounted) { + break; + } // Abort if unmounted during async operation setState((prev) => ({ ...prev, diff --git a/client/src/hooks/use-recording-session.ts b/client/src/hooks/use-recording-session.ts index de7b51d..0938ada 100644 --- a/client/src/hooks/use-recording-session.ts +++ b/client/src/hooks/use-recording-session.ts @@ -359,7 +359,9 @@ export function useRecordingSession( try { if (!isConnected && !shouldSimulate) { const connected = await preflightConnect(); - if (cancelled) return; + if (cancelled) { + return; + } if (!connected) { setRecordingState('idle'); return; @@ -368,7 +370,9 @@ export function useRecordingSession( if (!shouldSimulate) { await checkAndRecoverStream(); - if (cancelled) return; + if (cancelled) { + return; + } } const api: NoteFlowAPI = shouldSimulate && !isConnected ? mockAPI : getAPI(); @@ -377,7 +381,9 @@ export function useRecordingSession( include_segments: false, include_summary: false, }); - if (cancelled) return; + if (cancelled) { + return; + } setMeeting(existingMeeting); setMeetingTitle(existingMeeting.title); if (!['created', 'recording'].includes(existingMeeting.state)) { @@ -389,7 +395,9 @@ export function useRecordingSession( const mockModule: typeof import('@/api/mock-transcription-stream') = await import( '@/api/mock-transcription-stream' ); - if (cancelled) return; + if (cancelled) { + return; + } stream = new mockModule.MockTranscriptionStream(existingMeeting.id); } else { stream = ensureTranscriptionStream(await api.startTranscription(existingMeeting.id)); @@ -406,7 +414,9 @@ export function useRecordingSession( shouldSimulate ? 'Simulation is active' : 'Transcription is now active' ); } catch (error) { - if (cancelled) return; + if (cancelled) { + return; + } setRecordingState('idle'); toastError('Failed to start recording', error); } diff --git a/client/src/lib/ai-providers/model-catalog-utils.ts b/client/src/lib/ai-providers/model-catalog-utils.ts index 3b9e1da..c128128 100644 --- a/client/src/lib/ai-providers/model-catalog-utils.ts +++ b/client/src/lib/ai-providers/model-catalog-utils.ts @@ -37,7 +37,7 @@ function getStringOrNumber( function buildCostLabel(value: unknown): string | undefined { if (typeof value === 'string') { const trimmed = value.trim(); - return trimmed ? trimmed : undefined; + return trimmed || undefined; } if (typeof value === 'number' && Number.isFinite(value)) { return `$${value}`; @@ -70,7 +70,7 @@ function extractCostLabel(record: Record): string | undefined { } if (isRecord(record.pricing)) { - const pricing = record.pricing; + const {pricing} = record; const input = getStringOrNumber(pricing, ['prompt', 'input', 'input_cost', 'prompt_cost']); const output = getStringOrNumber(pricing, [ 'completion', diff --git a/client/src/lib/crypto.ts b/client/src/lib/crypto.ts index 296cc57..0916aa0 100644 --- a/client/src/lib/crypto.ts +++ b/client/src/lib/crypto.ts @@ -80,7 +80,9 @@ function getLegacyDeviceId(): string { async function decryptWithLegacyKey(encryptedString: string): Promise { try { const storedSalt = localStorage.getItem(KEY_SALT_KEY); - if (!storedSalt) return ''; + if (!storedSalt) { + return ''; + } const salt = new Uint8Array(JSON.parse(storedSalt)).buffer as ArrayBuffer; const legacyDeviceId = getLegacyDeviceId(); diff --git a/client/src/lib/log-messages.ts b/client/src/lib/log-messages.ts index 9cedf29..2cee431 100644 --- a/client/src/lib/log-messages.ts +++ b/client/src/lib/log-messages.ts @@ -207,7 +207,7 @@ const MESSAGE_TEMPLATES: Record = { 'reconnection callback execution failed': () => 'Reconnection partially completed', 'state sync after reconnect failed': () => 'Sync incomplete after reconnecting', 'scheduled reconnect attempt failed': (d) => { - const attempt = d.attempt; + const {attempt} = d; return attempt ? `Reconnection attempt ${attempt} failed` : 'Reconnection attempt failed'; }, 'online event reconnect failed': () => 'Could not reconnect when network came online', diff --git a/client/src/pages/Settings.tsx b/client/src/pages/Settings.tsx index 82c8b0c..103f530 100644 --- a/client/src/pages/Settings.tsx +++ b/client/src/pages/Settings.tsx @@ -329,7 +329,9 @@ export default function SettingsPage() { }; const handleServerSwitchConfirm = async () => { - if (!pendingServerChange) return; + if (!pendingServerChange) { + return; + } preferences.prepareForServerSwitch(); setIntegrations(preferences.getIntegrations()); await performConnect(pendingServerChange.host, pendingServerChange.port); diff --git a/docker/CLAUDE.md b/docker/CLAUDE.md new file mode 100644 index 0000000..2e56a77 --- /dev/null +++ b/docker/CLAUDE.md @@ -0,0 +1,1035 @@ +# Docker Security Rules for Claude Code + +These rules guide Claude Code to generate secure Docker configurations, Dockerfiles, and container deployments. Apply these rules when creating or modifying Docker-related files. + +--- + +## Rule: Minimal Base Images + +**Level**: `strict` + +**When**: Creating Dockerfiles or selecting base images + +**Do**: Use minimal base images appropriate for your application +```dockerfile +# Best for compiled languages (Go, Rust, C++) +FROM scratch AS runtime +COPY --from=builder /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ +COPY --from=builder /app/binary /binary +USER 65534:65534 +ENTRYPOINT ["/binary"] + +# Best for most applications - Google's distroless +FROM gcr.io/distroless/base-debian12:nonroot AS runtime +COPY --from=builder /app /app +USER nonroot:nonroot +ENTRYPOINT ["/app/server"] + +# Good for interpreted languages requiring packages +FROM python:3.12-alpine AS runtime +RUN apk add --no-cache ca-certificates tini && \ + rm -rf /var/cache/apk/* /tmp/* +USER nobody:nobody +ENTRYPOINT ["/sbin/tini", "--"] + +# For Java applications +FROM gcr.io/distroless/java21-debian12:nonroot +COPY --from=builder /app/app.jar /app/app.jar +USER nonroot:nonroot +ENTRYPOINT ["java", "-jar", "/app/app.jar"] +``` + +**Don't**: Use full OS images or unnecessarily large base images +```dockerfile +# Vulnerable: Full Ubuntu with unnecessary tools +FROM ubuntu:22.04 +RUN apt-get update && apt-get install -y \ + python3 python3-pip \ + curl wget vim nano \ + net-tools iputils-ping telnet \ + ssh-client \ + unzip tar gzip +# Problems: +# - 500+ packages with potential vulnerabilities +# - Includes shells for attackers +# - Package manager available for installing malware +# - Networking tools for reconnaissance + +# Vulnerable: Using :latest tag +FROM node:latest +# Problem: Unpredictable, may change and break builds +``` + +**Why**: Minimal images dramatically reduce attack surface. A typical Ubuntu image contains 100+ binaries that can be used for privilege escalation (`sudo`, `su`), lateral movement (`ssh`, `curl`, `wget`), or data exfiltration (`nc`, `tar`). Distroless images contain only the application runtime, reducing CVE count by 80-90%. + +**Refs**: CWE-250, CIS Docker Benchmark 4.1, NIST 800-190 Section 3.1 + +--- + +## Rule: Non-Root User Directive + +**Level**: `strict` + +**When**: Creating Dockerfiles + +**Do**: Create and switch to a non-root user +```dockerfile +FROM node:20-alpine + +# Create non-root user with specific UID for consistency across containers +RUN addgroup -g 10001 -S appgroup && \ + adduser -u 10001 -S -G appgroup -h /app -s /sbin/nologin appuser + +WORKDIR /app + +# Copy with correct ownership +COPY --chown=appuser:appgroup package*.json ./ +RUN npm ci --only=production && npm cache clean --force + +COPY --chown=appuser:appgroup . . + +# Switch to non-root user +USER appuser:appgroup + +EXPOSE 3000 +CMD ["node", "server.js"] +``` + +```dockerfile +# For Python applications +FROM python:3.12-alpine + +RUN addgroup -g 10001 -S appgroup && \ + adduser -u 10001 -S -G appgroup -h /app -s /sbin/nologin appuser + +WORKDIR /app + +COPY --chown=appuser:appgroup requirements.txt . +RUN pip install --no-cache-dir --user -r requirements.txt + +COPY --chown=appuser:appgroup . . + +USER appuser:appgroup + +ENV PATH="/app/.local/bin:$PATH" +EXPOSE 8000 +CMD ["python", "-m", "uvicorn", "main:app", "--host", "0.0.0.0", "--port", "8000"] +``` + +```dockerfile +# Using numeric UID for scratch/distroless +FROM golang:1.22-alpine AS builder +WORKDIR /build +COPY . . +RUN CGO_ENABLED=0 go build -ldflags="-s -w" -o /app + +FROM scratch +COPY --from=builder /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ +COPY --from=builder /app /app +# Use numeric UID (nobody = 65534) +USER 65534:65534 +ENTRYPOINT ["/app"] +``` + +**Don't**: Run containers as root +```dockerfile +# Vulnerable: No USER directive (runs as root) +FROM node:20 +WORKDIR /app +COPY . . +RUN npm install +EXPOSE 3000 +CMD ["node", "server.js"] +# Risk: Container escape = root access on host + +# Vulnerable: Explicitly running as root +USER root +RUN npm install +# Never switch back to non-root +CMD ["node", "server.js"] +``` + +**Why**: Container escape vulnerabilities like CVE-2019-5736 (runc) and CVE-2020-15257 (containerd) allow attackers to break out of containers. If the container runs as root (UID 0), the attacker gains root on the host. Running as non-root limits impact to unprivileged user access, significantly reducing the severity of container escapes. + +**Refs**: CWE-250, CWE-269, CIS Docker Benchmark 4.1, NIST 800-190 Section 4.2.1 + +--- + +## Rule: Multi-Stage Builds + +**Level**: `strict` + +**When**: Building application containers + +**Do**: Use multi-stage builds to separate build and runtime environments +```dockerfile +# Stage 1: Build environment with all build tools +FROM golang:1.22-alpine AS builder +RUN apk add --no-cache git ca-certificates tzdata + +WORKDIR /build +COPY go.mod go.sum ./ +RUN go mod download && go mod verify + +COPY . . +RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 \ + go build -ldflags="-s -w -X main.version=${VERSION}" \ + -o /app/server ./cmd/server + +# Stage 2: Minimal runtime +FROM gcr.io/distroless/static-debian12:nonroot +COPY --from=builder /usr/share/zoneinfo /usr/share/zoneinfo +COPY --from=builder /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ +COPY --from=builder /app/server /server +USER nonroot:nonroot +ENTRYPOINT ["/server"] +``` + +```dockerfile +# Node.js multi-stage build +FROM node:20-alpine AS builder +WORKDIR /build +COPY package*.json ./ +RUN npm ci +COPY . . +RUN npm run build && npm prune --production + +# Runtime stage +FROM node:20-alpine AS runtime +RUN apk add --no-cache tini && \ + addgroup -g 10001 -S appgroup && \ + adduser -u 10001 -S appuser -G appgroup + +WORKDIR /app +COPY --from=builder --chown=appuser:appgroup /build/dist ./dist +COPY --from=builder --chown=appuser:appgroup /build/node_modules ./node_modules +COPY --from=builder --chown=appuser:appgroup /build/package.json ./ + +USER appuser:appgroup +EXPOSE 3000 +ENTRYPOINT ["/sbin/tini", "--"] +CMD ["node", "dist/server.js"] +``` + +```dockerfile +# Python multi-stage with virtual environment +FROM python:3.12-alpine AS builder +RUN apk add --no-cache build-base libffi-dev +WORKDIR /build +COPY requirements.txt . +RUN python -m venv /opt/venv && \ + /opt/venv/bin/pip install --no-cache-dir -r requirements.txt + +FROM python:3.12-alpine AS runtime +RUN addgroup -g 10001 -S appgroup && \ + adduser -u 10001 -S appuser -G appgroup +COPY --from=builder /opt/venv /opt/venv +COPY --chown=appuser:appgroup . /app +WORKDIR /app +USER appuser:appgroup +ENV PATH="/opt/venv/bin:$PATH" +CMD ["python", "app.py"] +``` + +**Don't**: Include build tools in runtime images +```dockerfile +# Vulnerable: Build tools in runtime image +FROM node:20 +WORKDIR /app +COPY . . +RUN npm install && npm run build +# npm, build tools, dev dependencies all in final image +EXPOSE 3000 +CMD ["node", "dist/server.js"] +# Problems: +# - Includes npm (can install malware) +# - Includes dev dependencies (larger attack surface) +# - Source code visible in image +``` + +**Why**: Build environments contain compilers, package managers, development tools, and source code that aren't needed at runtime and increase attack surface. Attackers can use these tools to download malware, compile exploits, or exfiltrate data. Multi-stage builds produce minimal images with only runtime dependencies. + +**Refs**: CIS Docker Benchmark 4.9, NIST 800-190 Section 3.1 + +--- + +## Rule: No Secrets in Build Arguments or Layers + +**Level**: `strict` + +**When**: Handling secrets during container build or runtime + +**Do**: Use Docker BuildKit secrets or runtime injection +```dockerfile +# syntax=docker/dockerfile:1.4 + +FROM python:3.12-alpine AS builder + +# Mount secrets during build (not stored in layers) +RUN --mount=type=secret,id=pip_token \ + pip install --no-cache-dir \ + --extra-index-url https://$(cat /run/secrets/pip_token)@pypi.example.com/simple \ + -r requirements.txt + +FROM python:3.12-alpine AS runtime +COPY --from=builder /usr/local/lib/python3.12/site-packages /usr/local/lib/python3.12/site-packages +COPY . /app +USER nobody:nobody +CMD ["python", "/app/main.py"] +``` + +```bash +# Build with BuildKit secrets +DOCKER_BUILDKIT=1 docker build \ + --secret id=pip_token,src=./pip_token.txt \ + -t myapp:latest . +``` + +```dockerfile +# Runtime secret injection via environment (from orchestrator) +FROM python:3.12-alpine +COPY . /app +USER nobody:nobody +# Secrets injected at runtime by Docker/Kubernetes +CMD ["python", "/app/main.py"] +``` + +```yaml +# docker-compose.yml with secrets +version: '3.8' +services: + app: + build: . + secrets: + - db_password + - api_key + environment: + - DATABASE_PASSWORD_FILE=/run/secrets/db_password + - API_KEY_FILE=/run/secrets/api_key + +secrets: + db_password: + external: true + api_key: + external: true +``` + +**Don't**: Embed secrets in images +```dockerfile +# Vulnerable: Secrets in ARG (visible in history) +ARG DATABASE_PASSWORD +ENV DATABASE_PASSWORD=${DATABASE_PASSWORD} +# docker history shows the value + +# Vulnerable: Secrets in ENV +ENV API_KEY=sk-1234567890abcdef +# Plaintext in image configuration + +# Vulnerable: Copying secret files +COPY credentials.json /app/ +COPY .env /app/ +# Secrets extractable from image layers + +# Vulnerable: Secrets in RUN commands +RUN curl -H "Authorization: Bearer sk-secret123" https://api.example.com +# Visible in layer history +``` + +**Why**: Docker image layers are immutable and can be inspected. Secrets in ARG values appear in `docker history`. Secrets in ENV are stored in image config. Secrets in COPY persist in layers even if deleted later. Anyone with image access can extract these secrets. This violates secret management principles and makes rotation impossible. + +**Refs**: CWE-798, CWE-522, CIS Docker Benchmark 4.10, NIST 800-190 Section 4.2.3 + +--- + +## Rule: Image Vulnerability Scanning + +**Level**: `strict` + +**When**: Building and deploying Docker images + +**Do**: Integrate vulnerability scanning in CI/CD pipelines +```yaml +# GitHub Actions with Trivy +name: Build and Scan +on: [push, pull_request] + +jobs: + build-and-scan: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Build image + run: docker build -t myapp:${{ github.sha }} . + + - name: Scan for vulnerabilities + uses: aquasecurity/trivy-action@master + with: + image-ref: 'myapp:${{ github.sha }}' + format: 'table' + exit-code: '1' + severity: 'CRITICAL,HIGH' + ignore-unfixed: true + + - name: Scan for secrets + uses: aquasecurity/trivy-action@master + with: + image-ref: 'myapp:${{ github.sha }}' + scanners: 'secret' + exit-code: '1' + + - name: Scan Dockerfile + uses: aquasecurity/trivy-action@master + with: + scan-type: 'config' + scan-ref: '.' + exit-code: '1' + severity: 'CRITICAL,HIGH' +``` + +```bash +# Local scanning with Trivy +trivy image --severity CRITICAL,HIGH --exit-code 1 myapp:latest + +# Scan with vulnerability database update +trivy image --download-db-only +trivy image --skip-db-update myapp:latest + +# Scan for secrets and misconfigurations +trivy image --scanners vuln,secret,config myapp:latest + +# Generate SBOM +trivy image --format cyclonedx --output sbom.json myapp:latest + +# Scan with Grype +grype myapp:latest --fail-on high + +# Scan with Docker Scout +docker scout cves myapp:latest --exit-code --only-severity critical,high +``` + +```dockerfile +# Dockerfile best practices scanner +# hadolint Dockerfile +FROM python:3.12-alpine +# hadolint will flag issues like: +# - Using :latest tag +# - Missing USER directive +# - Curl/wget without verification +``` + +**Don't**: Deploy without vulnerability scanning +```yaml +# Vulnerable: No security scanning +jobs: + deploy: + steps: + - run: docker build -t myapp:latest . + - run: docker push myapp:latest + - run: kubectl set image deployment/app app=myapp:latest +# Risk: Critical CVEs deployed to production +``` + +**Why**: Container images contain vulnerabilities in base images, system packages, and application dependencies. Average images contain 50-200 vulnerabilities. Without scanning, critical vulnerabilities like remote code execution can be deployed. Automated scanning catches known CVEs before deployment and generates SBOMs for compliance. + +**Refs**: CWE-1104, NIST 800-190 Section 3.2, CIS Docker Benchmark 4.4 + +--- + +## Rule: Content Trust and Image Signing + +**Level**: `warning` + +**When**: Distributing or consuming Docker images + +**Do**: Sign images and enable content trust +```bash +# Enable Docker Content Trust +export DOCKER_CONTENT_TRUST=1 +export DOCKER_CONTENT_TRUST_SERVER=https://notary.example.com + +# Push will automatically sign +docker push myregistry.io/myapp:v1.0.0 + +# Pull will verify signature +docker pull myregistry.io/myapp:v1.0.0 + +# Sign with Cosign (recommended) +cosign generate-key-pair + +# Sign image +cosign sign --key cosign.key myregistry.io/myapp:v1.0.0 + +# Verify signature +cosign verify --key cosign.pub myregistry.io/myapp:v1.0.0 + +# Keyless signing with OIDC (GitHub Actions) +cosign sign --oidc-issuer=https://token.actions.githubusercontent.com \ + myregistry.io/myapp:v1.0.0 +``` + +```yaml +# GitHub Actions: Keyless signing with Cosign +- name: Sign image with Cosign + run: | + cosign sign --yes \ + --oidc-issuer=https://token.actions.githubusercontent.com \ + ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}@${{ steps.build.outputs.digest }} + env: + COSIGN_EXPERIMENTAL: "true" +``` + +**Don't**: Use unsigned images without verification +```bash +# Vulnerable: No signature verification +docker pull someregistry.io/app:latest +docker run someregistry.io/app:latest +# Risk: Image may be tampered or from malicious source + +# Vulnerable: Disabled content trust +export DOCKER_CONTENT_TRUST=0 +docker pull myregistry.io/myapp:latest +``` + +**Why**: Without signing, attackers can replace legitimate images through registry compromise, man-in-the-middle attacks, or typosquatting (e.g., `myap` vs `myapp`). Image signing provides cryptographic proof of origin and integrity. Content trust prevents pulling unsigned or tampered images. + +**Refs**: CWE-494, CIS Docker Benchmark 4.5, NIST 800-190 Section 3.3 + +--- + +## Rule: Read-Only Root Filesystem + +**Level**: `warning` + +**When**: Running Docker containers + +**Do**: Mount root filesystem as read-only with explicit writable directories +```bash +# Run with read-only root filesystem +docker run --read-only \ + --tmpfs /tmp:rw,noexec,nosuid,size=100m \ + --tmpfs /var/run:rw,noexec,nosuid,size=10m \ + -v app-logs:/var/log/app:rw \ + myapp:latest + +# Docker Compose +version: '3.8' +services: + app: + image: myapp:latest + read_only: true + tmpfs: + - /tmp:size=100m,mode=1777 + - /var/run:size=10m + volumes: + - app-logs:/var/log/app:rw + - app-data:/app/data:rw +``` + +```dockerfile +# Prepare application for read-only filesystem +FROM python:3.12-alpine + +WORKDIR /app +COPY . . +RUN pip install --no-cache-dir -r requirements.txt && \ + mkdir -p /app/tmp /app/logs /app/.cache && \ + chown -R nobody:nobody /app + +USER nobody:nobody + +# Configure app to use specific writable directories +ENV TMPDIR=/app/tmp +ENV CACHE_DIR=/app/.cache +CMD ["python", "app.py"] +``` + +**Don't**: Allow unrestricted filesystem writes +```bash +# Vulnerable: Writable filesystem +docker run myapp:latest +# Attacker can: +# - Modify application binaries +# - Install backdoors +# - Write malware +# - Modify configuration files +``` + +**Why**: A writable root filesystem allows attackers to modify application binaries, install backdoors, write malware, or tamper with configuration. Read-only filesystems prevent persistent modifications and limit the impact of application compromise. tmpfs mounts provide necessary writable space without persistence. + +**Refs**: CWE-284, CIS Docker Benchmark 5.12, NIST 800-190 Section 4.2.2 + +--- + +## Rule: Drop All Capabilities + +**Level**: `strict` + +**When**: Running Docker containers + +**Do**: Drop all Linux capabilities and add only required ones +```bash +# Drop all capabilities +docker run --cap-drop=ALL \ + --security-opt=no-new-privileges:true \ + myapp:latest + +# Add back specific capability if absolutely required +docker run --cap-drop=ALL \ + --cap-add=NET_BIND_SERVICE \ + --user 1000:1000 \ + --security-opt=no-new-privileges:true \ + myapp:latest +``` + +```yaml +# Docker Compose +version: '3.8' +services: + app: + image: myapp:latest + cap_drop: + - ALL + security_opt: + - no-new-privileges:true + # Only if binding to port < 1024 + # cap_add: + # - NET_BIND_SERVICE +``` + +**Common capabilities and their risks**: +```bash +# Dangerous capabilities to never add: +# CAP_SYS_ADMIN - Mount filesystems, load kernel modules +# CAP_SYS_PTRACE - Debug processes, bypass security +# CAP_SYS_RAWIO - Direct I/O access +# CAP_NET_ADMIN - Network configuration changes +# CAP_DAC_OVERRIDE - Bypass file permission checks + +# Rarely needed capabilities: +# CAP_NET_BIND_SERVICE - Bind to ports < 1024 +# CAP_CHOWN - Change file ownership +# CAP_SETUID - Change UID (dangerous!) +``` + +**Don't**: Run with default or elevated capabilities +```bash +# Vulnerable: Default capabilities (13 capabilities) +docker run myapp:latest + +# Critical: All capabilities +docker run --cap-add=ALL myapp:latest +# Equivalent to root on host + +# Vulnerable: Dangerous capabilities +docker run --cap-add=SYS_ADMIN myapp:latest +docker run --cap-add=SYS_PTRACE myapp:latest +``` + +**Why**: Linux capabilities divide root privileges into distinct units. Docker containers have 13 capabilities by default, including dangerous ones like CAP_NET_RAW (ARP spoofing, packet sniffing) and CAP_SETFCAP (setting file capabilities). Dropping all capabilities and adding only required ones follows least privilege and significantly reduces attack surface. + +**Refs**: CWE-250, CWE-269, CIS Docker Benchmark 5.3-5.4, NIST 800-190 Section 4.2.1 + +--- + +## Rule: No Privileged Containers + +**Level**: `strict` + +**When**: Running Docker containers + +**Do**: Run containers with restricted privileges +```bash +# Secure container runtime +docker run \ + --cap-drop=ALL \ + --security-opt=no-new-privileges:true \ + --security-opt=seccomp=default.json \ + --read-only \ + --user 10001:10001 \ + myapp:latest +``` + +```yaml +# Docker Compose with security options +version: '3.8' +services: + app: + image: myapp:latest + user: "10001:10001" + cap_drop: + - ALL + security_opt: + - no-new-privileges:true + - seccomp:seccomp-profile.json + read_only: true + tmpfs: + - /tmp +``` + +**Don't**: Run privileged containers +```bash +# Critical vulnerability: Privileged mode +docker run --privileged myapp:latest +# Gives full host access: +# - All capabilities +# - Access to all devices +# - Can mount host filesystem +# - Can load kernel modules +# - Effectively root on host + +# Vulnerable: Elevated privileges +docker run --cap-add=SYS_ADMIN myapp:latest +docker run --device=/dev/sda myapp:latest +docker run -v /:/host myapp:latest +``` + +**Why**: Privileged containers have full access to the host system, effectively negating container isolation. An attacker who compromises a privileged container has root access to the host and can escape the container trivially. This is the most severe container security misconfiguration. + +**Refs**: CWE-250, CWE-269, CIS Docker Benchmark 5.4, NIST 800-190 Section 4.2.1 + +--- + +## Rule: Container Health Checks + +**Level**: `advisory` + +**When**: Creating production Dockerfiles + +**Do**: Implement comprehensive health checks +```dockerfile +FROM python:3.12-alpine + +WORKDIR /app +COPY . . +RUN pip install --no-cache-dir -r requirements.txt + +USER nobody:nobody + +# Health check with appropriate intervals +HEALTHCHECK --interval=30s --timeout=10s --start-period=60s --retries=3 \ + CMD wget --no-verbose --tries=1 --spider http://localhost:8000/health || exit 1 + +EXPOSE 8000 +CMD ["python", "-m", "uvicorn", "main:app", "--host", "0.0.0.0", "--port", "8000"] +``` + +```dockerfile +# For containers without wget/curl +FROM gcr.io/distroless/base-debian12:nonroot + +COPY --from=builder /app/server /server +COPY --from=builder /app/healthcheck /healthcheck + +USER nonroot:nonroot + +HEALTHCHECK --interval=30s --timeout=10s --start-period=60s --retries=3 \ + CMD ["/healthcheck"] + +ENTRYPOINT ["/server"] +``` + +```dockerfile +# Multiple health check approaches +# HTTP endpoint +HEALTHCHECK CMD curl -f http://localhost:8080/health || exit 1 + +# TCP port check +HEALTHCHECK CMD nc -z localhost 5432 || exit 1 + +# Custom script +HEALTHCHECK CMD /app/healthcheck.sh || exit 1 + +# PostgreSQL +HEALTHCHECK CMD pg_isready -U postgres || exit 1 + +# Redis +HEALTHCHECK CMD redis-cli ping || exit 1 +``` + +**Don't**: Skip health checks in production +```dockerfile +# Vulnerable: No health check +FROM python:3.12-alpine +COPY . /app +CMD ["python", "/app/main.py"] +# Problems: +# - No automatic restart on failure +# - No readiness detection +# - Harder to detect compromised containers +``` + +**Why**: Without health checks, containers that crash, deadlock, or become unresponsive continue running. Orchestrators can't automatically restart failed containers or route traffic away from unhealthy instances. Health checks enable automatic recovery and can detect anomalies that may indicate compromise. + +**Refs**: CIS Docker Benchmark 4.6, NIST 800-190 Section 4.4.1 + +--- + +## Rule: Resource Limits + +**Level**: `warning` + +**When**: Running Docker containers in production + +**Do**: Set memory, CPU, and PID limits +```bash +# Run with resource limits +docker run \ + --memory="512m" \ + --memory-swap="512m" \ + --memory-reservation="256m" \ + --cpus="1.0" \ + --cpu-shares=1024 \ + --pids-limit=100 \ + --ulimit nofile=1024:1024 \ + --ulimit nproc=64:64 \ + myapp:latest +``` + +```yaml +# Docker Compose resource limits +version: '3.8' +services: + app: + image: myapp:latest + deploy: + resources: + limits: + cpus: '1.0' + memory: 512M + pids: 100 + reservations: + cpus: '0.5' + memory: 256M + ulimits: + nofile: + soft: 1024 + hard: 1024 + nproc: + soft: 64 + hard: 64 +``` + +```bash +# System-wide Docker daemon defaults +# /etc/docker/daemon.json +{ + "default-ulimits": { + "nofile": { + "Name": "nofile", + "Hard": 1024, + "Soft": 1024 + }, + "nproc": { + "Name": "nproc", + "Hard": 64, + "Soft": 64 + } + } +} +``` + +**Don't**: Run containers without resource limits +```bash +# Vulnerable: No limits +docker run myapp:latest +# Risks: +# - Fork bombs can exhaust PIDs +# - Memory leaks can OOM host +# - CPU mining can starve other containers +# - File descriptor exhaustion +``` + +**Why**: Containers without resource limits can consume all host resources, causing denial of service to other containers and the host system. Attackers exploit this through fork bombs (--pids-limit prevents), memory exhaustion (--memory prevents), and CPU abuse (--cpus prevents). Limits ensure fair resource sharing and contain compromise impact. + +**Refs**: CWE-400, CWE-770, CIS Docker Benchmark 5.10-5.14, NIST 800-190 Section 4.2.4 + +--- + +## Rule: Secure .dockerignore + +**Level**: `warning` + +**When**: Building Docker images + +**Do**: Create comprehensive .dockerignore to exclude sensitive files +```gitignore +# .dockerignore + +# Version control +.git +.gitignore +.svn + +# Secrets and credentials +.env +.env.* +*.pem +*.key +*.crt +**/secrets/ +credentials.json +service-account.json +*.secret + +# Build artifacts and dependencies +node_modules +__pycache__ +*.pyc +.pytest_cache +.coverage +coverage/ +dist/ +build/ +target/ + +# IDE and editor files +.vscode +.idea +*.swp +*.swo +*~ + +# Documentation and tests (usually not needed in runtime) +docs/ +*.md +README* +CHANGELOG* +test/ +tests/ +*_test.go +*.test.js + +# Docker files +Dockerfile* +docker-compose*.yml +.dockerignore + +# CI/CD +.github/ +.gitlab-ci.yml +.travis.yml +Jenkinsfile + +# OS files +.DS_Store +Thumbs.db + +# Log files +*.log +logs/ +``` + +**Don't**: Build images without .dockerignore +```bash +# Vulnerable: No .dockerignore +# The following get copied into the image: +# - .git directory (full history including deleted secrets) +# - .env files with credentials +# - Private keys and certificates +# - IDE settings with personal info +# - Test files and coverage reports +# - node_modules (may differ from npm ci) +``` + +**Why**: Without .dockerignore, COPY and ADD instructions include all files in the build context, including sensitive data like .git directories (containing full history), environment files with credentials, private keys, and IDE configurations. These become extractable from image layers and may be exposed if the image is published. + +**Refs**: CWE-200, CWE-522, CIS Docker Benchmark 4.10 + +--- + +## Additional Security Configurations + +### Seccomp Profile + +```json +{ + "defaultAction": "SCMP_ACT_ERRNO", + "architectures": ["SCMP_ARCH_X86_64", "SCMP_ARCH_AARCH64"], + "syscalls": [ + { + "names": [ + "read", "write", "open", "close", "stat", "fstat", + "lstat", "poll", "lseek", "mmap", "mprotect", "munmap", + "brk", "rt_sigaction", "rt_sigprocmask", "ioctl", + "access", "pipe", "select", "sched_yield", "mremap", + "msync", "mincore", "madvise", "shmget", "shmat", + "exit", "exit_group", "wait4", "kill", "uname", + "fcntl", "flock", "fsync", "fdatasync", "truncate", + "ftruncate", "getdents", "getcwd", "chdir", "fchdir", + "rename", "mkdir", "rmdir", "link", "unlink", "symlink", + "readlink", "chmod", "fchmod", "chown", "fchown", + "lchown", "umask", "gettimeofday", "getuid", "getgid", + "geteuid", "getegid", "getpgid", "getppid", "getpgrp", + "setsid", "setpgid", "getgroups", "setresuid", "setresgid", + "getresuid", "getresgid", "sigaltstack", "rt_sigreturn", + "clock_gettime", "clock_getres", "clock_nanosleep", + "futex", "sched_getaffinity", "epoll_create", "epoll_ctl", + "epoll_wait", "epoll_pwait", "epoll_create1", "dup", + "dup2", "dup3", "socket", "connect", "accept", "accept4", + "sendto", "recvfrom", "sendmsg", "recvmsg", "bind", + "listen", "getsockname", "getpeername", "socketpair", + "setsockopt", "getsockopt", "clone", "execve", "arch_prctl", + "prctl", "pread64", "pwrite64", "readv", "writev", + "getrandom", "memfd_create", "openat", "fstatat", "unlinkat", + "renameat", "faccessat", "fchmodat", "fchownat", "newfstatat" + ], + "action": "SCMP_ACT_ALLOW" + } + ] +} +``` + +### AppArmor Profile + +```bash +# /etc/apparmor.d/docker-myapp +#include + +profile docker-myapp flags=(attach_disconnected,mediate_deleted) { + #include + + network inet tcp, + network inet udp, + network inet icmp, + + deny @{PROC}/* w, + deny @{PROC}/sys/** w, + deny /sys/** w, + + /app/** r, + /app/logs/** rw, + /tmp/** rw, + + deny /etc/shadow r, + deny /etc/passwd r, +} +``` + +### Docker Daemon Security Configuration + +```json +{ + "icc": false, + "userns-remap": "default", + "no-new-privileges": true, + "seccomp-profile": "/etc/docker/seccomp-profile.json", + "live-restore": true, + "userland-proxy": false, + "log-driver": "json-file", + "log-opts": { + "max-size": "10m", + "max-file": "3" + }, + "storage-driver": "overlay2", + "default-ulimits": { + "nofile": { + "Name": "nofile", + "Hard": 65535, + "Soft": 65535 + } + } +} +``` + +**Refs**: CIS Docker Benchmark Section 2, NIST 800-190 Section 4 \ No newline at end of file diff --git a/docs/sprints/phase-2-intelligence/sprint-4-ner-extraction/README.md b/docs/sprints/phase-2-intelligence/sprint-4-ner-extraction/README.md index 7307bcd..2fe3564 100644 --- a/docs/sprints/phase-2-intelligence/sprint-4-ner-extraction/README.md +++ b/docs/sprints/phase-2-intelligence/sprint-4-ner-extraction/README.md @@ -1795,7 +1795,7 @@ from uuid import uuid4 import pytest -from noteflow.application.services.ner_service import ExtractionResult, NerService +from noteflow.application.services.ner import ExtractionResult, NerService from noteflow.domain.entities.meeting import Meeting from noteflow.domain.entities.named_entity import EntityCategory, NamedEntity from noteflow.domain.entities.segment import Segment diff --git a/docs/sprints/phase-3-integrations/sprint-6-webhooks/README.md b/docs/sprints/phase-3-integrations/sprint-6-webhooks/README.md index 057f05c..34e3d23 100644 --- a/docs/sprints/phase-3-integrations/sprint-6-webhooks/README.md +++ b/docs/sprints/phase-3-integrations/sprint-6-webhooks/README.md @@ -1020,7 +1020,7 @@ The NoteFlow gRPC layer uses a three-tier service injection pattern. All three t ```python # Add to ServicerHost protocol class: -from noteflow.application.services.webhook_service import WebhookService +from noteflow.application.services.webhooks import WebhookService class ServicerHost(Protocol): # ... existing fields ... @@ -1034,7 +1034,7 @@ class ServicerHost(Protocol): **File**: `src/noteflow/grpc/service.py` ```python -from noteflow.application.services.webhook_service import WebhookService +from noteflow.application.services.webhooks import WebhookService class NoteFlowServicer( # ... existing mixins ... @@ -1053,7 +1053,7 @@ class NoteFlowServicer( **File**: `src/noteflow/grpc/server.py` ```python -from noteflow.application.services.webhook_service import WebhookService +from noteflow.application.services.webhooks import WebhookService class NoteFlowServer: def __init__( @@ -1127,7 +1127,7 @@ if self._webhook_service is not None: Add to `run_server_with_config()` function: ```python -from noteflow.application.services.webhook_service import WebhookService +from noteflow.application.services.webhooks import WebhookService from noteflow.infrastructure.integrations.webhooks import WebhookExecutor from noteflow.infrastructure.converters import WebhookConverter @@ -1457,7 +1457,7 @@ def test_hmac_signature_generation(webhook_config: WebhookConfig) -> None: import pytest from unittest.mock import AsyncMock, Mock -from noteflow.application.services.webhook_service import WebhookService +from noteflow.application.services.webhooks import WebhookService from noteflow.domain.entities.meeting import Meeting, MeetingId from noteflow.domain.webhooks.events import WebhookConfig, WebhookEventType diff --git a/docs/sprints/phase-ongoing/.archive/sprint-gap-011-post-processing-pipeline/TESTING.md b/docs/sprints/phase-ongoing/.archive/sprint-gap-011-post-processing-pipeline/TESTING.md index 68df088..27aadd1 100644 --- a/docs/sprints/phase-ongoing/.archive/sprint-gap-011-post-processing-pipeline/TESTING.md +++ b/docs/sprints/phase-ongoing/.archive/sprint-gap-011-post-processing-pipeline/TESTING.md @@ -633,7 +633,7 @@ import pytest from unittest.mock import AsyncMock, MagicMock from noteflow.domain.webhooks.events import WebhookEventType -from noteflow.application.services.webhook_service import WebhookService +from noteflow.application.services.webhooks import WebhookService class TestNewWebhookEvents: diff --git a/docs/sprints/phase-ongoing/.archive/sprint_04_constant_imports.md b/docs/sprints/phase-ongoing/.archive/sprint_04_constant_imports.md index df5a6b8..9b9f69e 100644 --- a/docs/sprints/phase-ongoing/.archive/sprint_04_constant_imports.md +++ b/docs/sprints/phase-ongoing/.archive/sprint_04_constant_imports.md @@ -233,7 +233,7 @@ modules = [ 'noteflow.grpc._startup', 'noteflow.grpc.service', 'noteflow.grpc.server', - 'noteflow.application.services.export_service', + 'noteflow.application.services.export', ] for mod in modules: try: diff --git a/repomix.config.json b/repomix.config.json index 1f71459..80d16e2 100644 --- a/repomix.config.json +++ b/repomix.config.json @@ -27,7 +27,7 @@ } }, "include": [ - "src/", "client/src-tauri", "client/src", "tests/" + "src/" ], "ignore": { "useGitignore": true, diff --git a/scripts/ab_streaming_harness.py b/scripts/ab_streaming_harness.py index 5c59031..d256099 100644 --- a/scripts/ab_streaming_harness.py +++ b/scripts/ab_streaming_harness.py @@ -25,7 +25,7 @@ from noteflow.infrastructure.security.crypto import AesGcmCryptoBox from noteflow.infrastructure.security.keystore import KeyringKeyStore if TYPE_CHECKING: - from noteflow.grpc._types import TranscriptSegment + from noteflow.grpc.types import TranscriptSegment PRESETS: dict[str, dict[str, float]] = { "responsive": { diff --git a/src/AGENTS.md b/src/AGENTS.md new file mode 100644 index 0000000..a020518 --- /dev/null +++ b/src/AGENTS.md @@ -0,0 +1,419 @@ +# Python Security & Coding Guidelines + +Security rules and coding practices for Python development. + +## Prerequisites +- `rules/_core/owasp-2025.md` - Core web security +- `rules/_core/ai-security.md` - AI/ML security (if applicable) + +--- + +## Input Handling + +### Avoid Dangerous Deserialization +**Level**: `strict` | **When**: Loading untrusted data | **Refs**: CWE-502, OWASP A08:2025 + +**Do**: +```python +import json, yaml +data = json.loads(user_input) # Safe +data = yaml.safe_load(user_input) # Safe loader +``` + +**Don't**: +```python +import pickle, yaml +data = pickle.loads(user_input) # RCE vulnerability +data = yaml.load(user_input, Loader=yaml.Loader) # Arbitrary code execution +``` + +**Why**: Pickle and unsafe YAML loaders execute arbitrary code during deserialization. + +--- + +### Use Subprocess Safely +**Level**: `strict` | **When**: Executing system commands | **Refs**: CWE-78, OWASP A03:2025 + +**Do**: +```python +import subprocess, shlex, re +# Pass arguments as list (no shell) +result = subprocess.run(['ls', '-la', user_dir], capture_output=True, text=True, check=True) +# If shell=True required, validate strictly +if re.match(r'^[a-zA-Z0-9_-]+$', filename): + subprocess.run(f'process {shlex.quote(filename)}', shell=True) +``` + +**Don't**: +```python +import os, subprocess +os.system(f'ls {user_input}') # Command injection +subprocess.run(f'grep {pattern} {filename}', shell=True) # Shell injection +``` + +**Why**: Shell injection allows attackers to execute arbitrary commands. + +--- + +## File Operations + +### Prevent Path Traversal +**Level**: `strict` | **When**: File access based on user input | **Refs**: CWE-22, OWASP A01:2025 + +**Do**: +```python +from pathlib import Path +UPLOAD_DIR = Path('/app/uploads').resolve() +def safe_file_access(filename: str) -> Path: + requested = (UPLOAD_DIR / filename).resolve() + if not requested.is_relative_to(UPLOAD_DIR): + raise ValueError("Path traversal attempt detected") + return requested +``` + +**Don't**: +```python +def get_file(filename): + return open(f'/app/uploads/{filename}').read() # Path traversal vulnerability +path = os.path.join(base_dir, user_filename) # No validation +``` + +**Why**: Path traversal (../) allows reading arbitrary files like /etc/passwd. + +--- + +### Secure Temporary Files +**Level**: `warning` | **When**: Creating temporary files | **Refs**: CWE-377, CWE-379 + +**Do**: +```python +import tempfile, os +with tempfile.NamedTemporaryFile(delete=True) as tmp: + tmp.write(data); tmp.flush(); process_file(tmp.name) +with tempfile.TemporaryDirectory() as tmpdir: + filepath = os.path.join(tmpdir, 'data.txt') +``` + +**Don't**: +```python +tmp_file = f'/tmp/myapp_{user_id}.txt' # Predictable filename, race condition +with open(tmp_file, 'w') as f: f.write(data) +``` + +**Why**: Predictable temp file names enable symlink attacks and race conditions. + +--- + +## Cryptography + +### Use Secure Random Numbers +**Level**: `strict` | **When**: Generating security-sensitive random values | **Refs**: CWE-330, CWE-338 + +**Do**: +```python +import secrets +token = secrets.token_urlsafe(32) # Secure token +api_key = secrets.token_hex(32) # Secure API key +otp = ''.join(secrets.choice('0123456789') for _ in range(6)) # Secure OTP +``` + +**Don't**: +```python +import random +token = ''.join(random.choices('abcdef0123456789', k=32)) # Predictable +session_id = random.randint(0, 999999) # Predictable PRNG +``` + +**Why**: `random` module uses predictable PRNG; attackers can predict tokens. + +--- + +### Hash Passwords Correctly +**Level**: `strict` | **When**: Storing user passwords | **Refs**: CWE-916, CWE-328, OWASP A02:2025 + +**Do**: +```python +import bcrypt +def hash_password(password: str) -> bytes: + return bcrypt.hashpw(password.encode(), bcrypt.gensalt(rounds=12)) +def verify_password(password: str, hashed: bytes) -> bool: + return bcrypt.checkpw(password.encode(), hashed) +``` + +**Don't**: +```python +import hashlib +password_hash = hashlib.sha256(password.encode()).hexdigest() # No salt, fast hash +password_hash = hashlib.md5(password.encode()).hexdigest() # MD5 is broken +``` + +**Why**: Unsalted fast hashes are vulnerable to rainbow tables and GPU cracking. + +--- + +## SQL Security + +### Use Parameterized Queries +**Level**: `strict` | **When**: Database queries with user input | **Refs**: CWE-89, OWASP A03:2025 + +**Do**: +```python +import sqlite3 +cursor.execute("SELECT * FROM users WHERE email = ? AND status = ?", (email, status)) +cursor.execute("SELECT * FROM users WHERE email = :email", {"email": email}) +user = session.query(User).filter(User.email == email).first() # SQLAlchemy ORM +``` + +**Don't**: +```python +query = f"SELECT * FROM users WHERE email = '{email}'" # SQL injection +cursor.execute(query) +cursor.execute("SELECT * FROM users WHERE id = %s" % user_id) # String formatting +``` + +**Why**: SQL injection allows attackers to read, modify, or delete database data. + +--- + +## Web Security + +### Validate URL Schemes +**Level**: `strict` | **When**: Processing user-provided URLs | **Refs**: CWE-918, OWASP A10:2025 + +**Do**: +```python +from urllib.parse import urlparse +ALLOWED_SCHEMES = {'http', 'https'} +def validate_url(url: str) -> str: + parsed = urlparse(url) + if parsed.scheme not in ALLOWED_SCHEMES: + raise ValueError(f"Invalid scheme: {parsed.scheme}") + if not parsed.netloc: + raise ValueError("Missing hostname") + return url +``` + +**Don't**: +```python +def fetch_url(url): + return requests.get(url) # Allows file://, javascript:, etc. +``` + +**Why**: Malicious schemes like `file://` or `javascript:` can read local files or execute code. + +--- + +### Set Secure Cookie Attributes +**Level**: `strict` | **When**: Setting cookies in web applications | **Refs**: CWE-614, CWE-1004, OWASP A07:2025 + +**Do**: +```python +from flask import make_response +response = make_response(data) +response.set_cookie('session_id', value=session_id, httponly=True, secure=True, samesite='Lax', max_age=3600) +``` + +**Don't**: +```python +response.set_cookie('session_id', session_id) # Missing security attributes +``` + +**Why**: Missing attributes expose cookies to XSS theft and CSRF attacks. + +--- + +## Error Handling + +### Don't Expose Stack Traces +**Level**: `warning` | **When**: Handling exceptions in production | **Refs**: CWE-209, OWASP A05:2025 + +**Do**: +```python +import logging +logger = logging.getLogger(__name__) +@app.errorhandler(Exception) +def handle_error(error): + logger.exception("Unhandled exception") # Log full details internally + return {"error": "Internal server error"}, 500 # Return safe message to client +``` + +**Don't**: +```python +@app.errorhandler(Exception) +def handle_error(error): + return {"error": str(error), "traceback": traceback.format_exc()}, 500 # Exposes internals +``` + +**Why**: Stack traces reveal file paths, library versions, and code structure to attackers. + +--- + +## Quick Reference + +| Rule | Level | CWE | +|------|-------|-----| +| Avoid pickle/unsafe YAML | strict | CWE-502 | +| Safe subprocess | strict | CWE-78 | +| Path traversal prevention | strict | CWE-22 | +| Secure temp files | warning | CWE-377 | +| Cryptographic randomness | strict | CWE-330 | +| Password hashing | strict | CWE-916 | +| Parameterized queries | strict | CWE-89 | +| URL scheme validation | strict | CWE-918 | +| Secure cookies | strict | CWE-614 | +| No stack traces | warning | CWE-209 | + +--- + +## Python Coding Practices + +### 1. Look Before You Leap (LBYL) +Check conditions proactively rather than using exceptions for control flow. + +```python +# WRONG: Exception as control flow +try: value = mapping[key]; process(value) +except KeyError: pass + +# CORRECT: Check first +if key in mapping: value = mapping[key]; process(value) +``` + +### 2. Never Swallow Exceptions +Avoid silent exception swallowing that hides critical failures. + +```python +# WRONG: Silent exception swallowing +try: risky_operation() +except: pass + +# CORRECT: Let exceptions bubble up +risky_operation() +``` + +### 3. Magic Methods Must Be O(1) +Magic methods called frequently must run in constant time. + +```python +# WRONG: __len__ doing iteration +def __len__(self) -> int: return sum(1 for _ in self._items) + +# CORRECT: O(1) __len__ +def __len__(self) -> int: return self._count +``` + +### 4. Check Existence Before Resolution +Always check `.exists()` before calling `.resolve()` or `.is_relative_to()`. + +```python +# WRONG: resolve() can raise OSError on non-existent paths +wt_path_resolved = wt_path.resolve() +if current_dir.is_relative_to(wt_path_resolved): current_worktree = wt_path_resolved + +# CORRECT: Check exists() first +if wt_path.exists(): + wt_path_resolved = wt_path.resolve() + if current_dir.is_relative_to(wt_path_resolved): current_worktree = wt_path_resolved +``` + +### 5. Defer Import-Time Computation +Avoid side effects at import time using `@cache`. + +```python +# WRONG: Path computed at import time +SESSION_FILE = Path("scratch/current-session-id") + +# CORRECT: Defer with @cache +@cache +def _session_file_path() -> Path: + """Return path to session ID file (cached after first call).""" + return Path("scratch/current-session-id") +``` + +### 6. Verify Casts at Runtime +Add assertions before `typing.cast()` calls. + +```python +# WRONG: Blind cast +cast(dict[str, Any], doc)["key"] = value + +# CORRECT: Assert before cast +assert isinstance(doc, MutableMapping), f"Expected MutableMapping, got {type(doc)}" +cast(dict[str, Any], doc)["key"] = value +``` + +### 7. Use Literal Types for Fixed Values +Model fixed string values with `Literal` types. + +```python +# WRONG: Bare strings +issues.append(("orphen-state", "desc")) # Typo goes unnoticed! + +# CORRECT: Literal type +IssueCode = Literal["orphan-state", "orphan-dir", "missing-branch"] +@dataclass(frozen=True) +class Issue: code: IssueCode; message: str +issues.append(Issue(code="orphan-state", message="desc")) # Type-checked! +``` + +### 8. Declare Variables Close to Use +Avoid early declarations that pollute scope. + +```python +# WRONG: Variable declared far from use +def process_data(ctx, items): + result_path = compute_result_path(ctx) + # ... 20 lines of other logic ... + save_to_path(transformed, result_path) + +# CORRECT: Inline at call site +def process_data(ctx, items): + # ... other logic ... + save_to_path(transformed, compute_result_path(ctx)) +``` + +### 9. Keyword Arguments for Complex Functions +Use keyword-only arguments for functions with 5+ parameters. + +```python +# WRONG: Positional chaos +response = fetch_data(api_url, 30.0, 3, {"Accept": "application/json"}, token) + +# CORRECT: Keyword-only after first param +def fetch_data(url, *, timeout: float, retries: int, headers: dict[str, str], auth_token: str) -> Response: ... +response = fetch_data(api_url, timeout=30.0, retries=3, headers={"Accept": "application/json"}, auth_token=token) +``` + +### 10. Default Values Are Dangerous +Require explicit parameter choices unless defaults are truly necessary. + +```python +# DANGEROUS: Caller forgets encoding +def process_file(path: Path, encoding: str = "utf-8") -> str: + return path.read_text(encoding=encoding) + +# SAFER: Require explicit choice +def process_file(path: Path, encoding: str) -> str: + return path.read_text(encoding=encoding) +``` + +--- + +## Quick Reference + +| Rule | Level | CWE | +|------|-------|-----| +| Avoid pickle/unsafe YAML | strict | CWE-502 | +| Safe subprocess | strict | CWE-78 | +| Path traversal prevention | strict | CWE-22 | +| Secure temp files | warning | CWE-377 | +| Cryptographic randomness | strict | CWE-330 | +| Password hashing | strict | CWE-916 | +| Parameterized queries | strict | CWE-89 | +| URL scheme validation | strict | CWE-918 | +| Secure cookies | strict | CWE-614 | +| No stack traces | warning | CWE-209 | + +--- + + diff --git a/src/CLAUDE.md b/src/CLAUDE.md new file mode 100644 index 0000000..a020518 --- /dev/null +++ b/src/CLAUDE.md @@ -0,0 +1,419 @@ +# Python Security & Coding Guidelines + +Security rules and coding practices for Python development. + +## Prerequisites +- `rules/_core/owasp-2025.md` - Core web security +- `rules/_core/ai-security.md` - AI/ML security (if applicable) + +--- + +## Input Handling + +### Avoid Dangerous Deserialization +**Level**: `strict` | **When**: Loading untrusted data | **Refs**: CWE-502, OWASP A08:2025 + +**Do**: +```python +import json, yaml +data = json.loads(user_input) # Safe +data = yaml.safe_load(user_input) # Safe loader +``` + +**Don't**: +```python +import pickle, yaml +data = pickle.loads(user_input) # RCE vulnerability +data = yaml.load(user_input, Loader=yaml.Loader) # Arbitrary code execution +``` + +**Why**: Pickle and unsafe YAML loaders execute arbitrary code during deserialization. + +--- + +### Use Subprocess Safely +**Level**: `strict` | **When**: Executing system commands | **Refs**: CWE-78, OWASP A03:2025 + +**Do**: +```python +import subprocess, shlex, re +# Pass arguments as list (no shell) +result = subprocess.run(['ls', '-la', user_dir], capture_output=True, text=True, check=True) +# If shell=True required, validate strictly +if re.match(r'^[a-zA-Z0-9_-]+$', filename): + subprocess.run(f'process {shlex.quote(filename)}', shell=True) +``` + +**Don't**: +```python +import os, subprocess +os.system(f'ls {user_input}') # Command injection +subprocess.run(f'grep {pattern} {filename}', shell=True) # Shell injection +``` + +**Why**: Shell injection allows attackers to execute arbitrary commands. + +--- + +## File Operations + +### Prevent Path Traversal +**Level**: `strict` | **When**: File access based on user input | **Refs**: CWE-22, OWASP A01:2025 + +**Do**: +```python +from pathlib import Path +UPLOAD_DIR = Path('/app/uploads').resolve() +def safe_file_access(filename: str) -> Path: + requested = (UPLOAD_DIR / filename).resolve() + if not requested.is_relative_to(UPLOAD_DIR): + raise ValueError("Path traversal attempt detected") + return requested +``` + +**Don't**: +```python +def get_file(filename): + return open(f'/app/uploads/{filename}').read() # Path traversal vulnerability +path = os.path.join(base_dir, user_filename) # No validation +``` + +**Why**: Path traversal (../) allows reading arbitrary files like /etc/passwd. + +--- + +### Secure Temporary Files +**Level**: `warning` | **When**: Creating temporary files | **Refs**: CWE-377, CWE-379 + +**Do**: +```python +import tempfile, os +with tempfile.NamedTemporaryFile(delete=True) as tmp: + tmp.write(data); tmp.flush(); process_file(tmp.name) +with tempfile.TemporaryDirectory() as tmpdir: + filepath = os.path.join(tmpdir, 'data.txt') +``` + +**Don't**: +```python +tmp_file = f'/tmp/myapp_{user_id}.txt' # Predictable filename, race condition +with open(tmp_file, 'w') as f: f.write(data) +``` + +**Why**: Predictable temp file names enable symlink attacks and race conditions. + +--- + +## Cryptography + +### Use Secure Random Numbers +**Level**: `strict` | **When**: Generating security-sensitive random values | **Refs**: CWE-330, CWE-338 + +**Do**: +```python +import secrets +token = secrets.token_urlsafe(32) # Secure token +api_key = secrets.token_hex(32) # Secure API key +otp = ''.join(secrets.choice('0123456789') for _ in range(6)) # Secure OTP +``` + +**Don't**: +```python +import random +token = ''.join(random.choices('abcdef0123456789', k=32)) # Predictable +session_id = random.randint(0, 999999) # Predictable PRNG +``` + +**Why**: `random` module uses predictable PRNG; attackers can predict tokens. + +--- + +### Hash Passwords Correctly +**Level**: `strict` | **When**: Storing user passwords | **Refs**: CWE-916, CWE-328, OWASP A02:2025 + +**Do**: +```python +import bcrypt +def hash_password(password: str) -> bytes: + return bcrypt.hashpw(password.encode(), bcrypt.gensalt(rounds=12)) +def verify_password(password: str, hashed: bytes) -> bool: + return bcrypt.checkpw(password.encode(), hashed) +``` + +**Don't**: +```python +import hashlib +password_hash = hashlib.sha256(password.encode()).hexdigest() # No salt, fast hash +password_hash = hashlib.md5(password.encode()).hexdigest() # MD5 is broken +``` + +**Why**: Unsalted fast hashes are vulnerable to rainbow tables and GPU cracking. + +--- + +## SQL Security + +### Use Parameterized Queries +**Level**: `strict` | **When**: Database queries with user input | **Refs**: CWE-89, OWASP A03:2025 + +**Do**: +```python +import sqlite3 +cursor.execute("SELECT * FROM users WHERE email = ? AND status = ?", (email, status)) +cursor.execute("SELECT * FROM users WHERE email = :email", {"email": email}) +user = session.query(User).filter(User.email == email).first() # SQLAlchemy ORM +``` + +**Don't**: +```python +query = f"SELECT * FROM users WHERE email = '{email}'" # SQL injection +cursor.execute(query) +cursor.execute("SELECT * FROM users WHERE id = %s" % user_id) # String formatting +``` + +**Why**: SQL injection allows attackers to read, modify, or delete database data. + +--- + +## Web Security + +### Validate URL Schemes +**Level**: `strict` | **When**: Processing user-provided URLs | **Refs**: CWE-918, OWASP A10:2025 + +**Do**: +```python +from urllib.parse import urlparse +ALLOWED_SCHEMES = {'http', 'https'} +def validate_url(url: str) -> str: + parsed = urlparse(url) + if parsed.scheme not in ALLOWED_SCHEMES: + raise ValueError(f"Invalid scheme: {parsed.scheme}") + if not parsed.netloc: + raise ValueError("Missing hostname") + return url +``` + +**Don't**: +```python +def fetch_url(url): + return requests.get(url) # Allows file://, javascript:, etc. +``` + +**Why**: Malicious schemes like `file://` or `javascript:` can read local files or execute code. + +--- + +### Set Secure Cookie Attributes +**Level**: `strict` | **When**: Setting cookies in web applications | **Refs**: CWE-614, CWE-1004, OWASP A07:2025 + +**Do**: +```python +from flask import make_response +response = make_response(data) +response.set_cookie('session_id', value=session_id, httponly=True, secure=True, samesite='Lax', max_age=3600) +``` + +**Don't**: +```python +response.set_cookie('session_id', session_id) # Missing security attributes +``` + +**Why**: Missing attributes expose cookies to XSS theft and CSRF attacks. + +--- + +## Error Handling + +### Don't Expose Stack Traces +**Level**: `warning` | **When**: Handling exceptions in production | **Refs**: CWE-209, OWASP A05:2025 + +**Do**: +```python +import logging +logger = logging.getLogger(__name__) +@app.errorhandler(Exception) +def handle_error(error): + logger.exception("Unhandled exception") # Log full details internally + return {"error": "Internal server error"}, 500 # Return safe message to client +``` + +**Don't**: +```python +@app.errorhandler(Exception) +def handle_error(error): + return {"error": str(error), "traceback": traceback.format_exc()}, 500 # Exposes internals +``` + +**Why**: Stack traces reveal file paths, library versions, and code structure to attackers. + +--- + +## Quick Reference + +| Rule | Level | CWE | +|------|-------|-----| +| Avoid pickle/unsafe YAML | strict | CWE-502 | +| Safe subprocess | strict | CWE-78 | +| Path traversal prevention | strict | CWE-22 | +| Secure temp files | warning | CWE-377 | +| Cryptographic randomness | strict | CWE-330 | +| Password hashing | strict | CWE-916 | +| Parameterized queries | strict | CWE-89 | +| URL scheme validation | strict | CWE-918 | +| Secure cookies | strict | CWE-614 | +| No stack traces | warning | CWE-209 | + +--- + +## Python Coding Practices + +### 1. Look Before You Leap (LBYL) +Check conditions proactively rather than using exceptions for control flow. + +```python +# WRONG: Exception as control flow +try: value = mapping[key]; process(value) +except KeyError: pass + +# CORRECT: Check first +if key in mapping: value = mapping[key]; process(value) +``` + +### 2. Never Swallow Exceptions +Avoid silent exception swallowing that hides critical failures. + +```python +# WRONG: Silent exception swallowing +try: risky_operation() +except: pass + +# CORRECT: Let exceptions bubble up +risky_operation() +``` + +### 3. Magic Methods Must Be O(1) +Magic methods called frequently must run in constant time. + +```python +# WRONG: __len__ doing iteration +def __len__(self) -> int: return sum(1 for _ in self._items) + +# CORRECT: O(1) __len__ +def __len__(self) -> int: return self._count +``` + +### 4. Check Existence Before Resolution +Always check `.exists()` before calling `.resolve()` or `.is_relative_to()`. + +```python +# WRONG: resolve() can raise OSError on non-existent paths +wt_path_resolved = wt_path.resolve() +if current_dir.is_relative_to(wt_path_resolved): current_worktree = wt_path_resolved + +# CORRECT: Check exists() first +if wt_path.exists(): + wt_path_resolved = wt_path.resolve() + if current_dir.is_relative_to(wt_path_resolved): current_worktree = wt_path_resolved +``` + +### 5. Defer Import-Time Computation +Avoid side effects at import time using `@cache`. + +```python +# WRONG: Path computed at import time +SESSION_FILE = Path("scratch/current-session-id") + +# CORRECT: Defer with @cache +@cache +def _session_file_path() -> Path: + """Return path to session ID file (cached after first call).""" + return Path("scratch/current-session-id") +``` + +### 6. Verify Casts at Runtime +Add assertions before `typing.cast()` calls. + +```python +# WRONG: Blind cast +cast(dict[str, Any], doc)["key"] = value + +# CORRECT: Assert before cast +assert isinstance(doc, MutableMapping), f"Expected MutableMapping, got {type(doc)}" +cast(dict[str, Any], doc)["key"] = value +``` + +### 7. Use Literal Types for Fixed Values +Model fixed string values with `Literal` types. + +```python +# WRONG: Bare strings +issues.append(("orphen-state", "desc")) # Typo goes unnoticed! + +# CORRECT: Literal type +IssueCode = Literal["orphan-state", "orphan-dir", "missing-branch"] +@dataclass(frozen=True) +class Issue: code: IssueCode; message: str +issues.append(Issue(code="orphan-state", message="desc")) # Type-checked! +``` + +### 8. Declare Variables Close to Use +Avoid early declarations that pollute scope. + +```python +# WRONG: Variable declared far from use +def process_data(ctx, items): + result_path = compute_result_path(ctx) + # ... 20 lines of other logic ... + save_to_path(transformed, result_path) + +# CORRECT: Inline at call site +def process_data(ctx, items): + # ... other logic ... + save_to_path(transformed, compute_result_path(ctx)) +``` + +### 9. Keyword Arguments for Complex Functions +Use keyword-only arguments for functions with 5+ parameters. + +```python +# WRONG: Positional chaos +response = fetch_data(api_url, 30.0, 3, {"Accept": "application/json"}, token) + +# CORRECT: Keyword-only after first param +def fetch_data(url, *, timeout: float, retries: int, headers: dict[str, str], auth_token: str) -> Response: ... +response = fetch_data(api_url, timeout=30.0, retries=3, headers={"Accept": "application/json"}, auth_token=token) +``` + +### 10. Default Values Are Dangerous +Require explicit parameter choices unless defaults are truly necessary. + +```python +# DANGEROUS: Caller forgets encoding +def process_file(path: Path, encoding: str = "utf-8") -> str: + return path.read_text(encoding=encoding) + +# SAFER: Require explicit choice +def process_file(path: Path, encoding: str) -> str: + return path.read_text(encoding=encoding) +``` + +--- + +## Quick Reference + +| Rule | Level | CWE | +|------|-------|-----| +| Avoid pickle/unsafe YAML | strict | CWE-502 | +| Safe subprocess | strict | CWE-78 | +| Path traversal prevention | strict | CWE-22 | +| Secure temp files | warning | CWE-377 | +| Cryptographic randomness | strict | CWE-330 | +| Password hashing | strict | CWE-916 | +| Parameterized queries | strict | CWE-89 | +| URL scheme validation | strict | CWE-918 | +| Secure cookies | strict | CWE-614 | +| No stack traces | warning | CWE-209 | + +--- + + diff --git a/src/noteflow/CLAUDE.md b/src/noteflow/CLAUDE.md new file mode 100644 index 0000000..db18663 --- /dev/null +++ b/src/noteflow/CLAUDE.md @@ -0,0 +1,608 @@ +# CLAUDE.md - Python Backend Reference + +This file provides guidance for working with the Python backend code in `src/noteflow/`. For client-side development, see the main repository CLAUDE.md. + +## Project Overview + +NoteFlow is an intelligent meeting notetaker: local-first audio capture + navigable recall + evidence-linked summaries. This is the **Python backend** (`src/noteflow/`) — gRPC server, domain logic, infrastructure adapters. + +The gRPC schema is the shared contract between backend and client; keep proto changes in sync across Python, Rust, and TypeScript. + +--- + +## Quick Orientation + +| Entry Point | Description | +|-------------|-------------| +| `python -m noteflow.grpc.server` | Backend server (`src/noteflow/grpc/server/__main__.py`) | +| `src/noteflow/grpc/proto/noteflow.proto` | Protobuf schema | + +--- + +## Build and Development Commands + +```bash +# Install (editable with dev dependencies) +python -m pip install -e ".[dev]" + +# Run gRPC server +python -m noteflow.grpc.server --help + +# Tests +pytest # Full suite +pytest -m "not integration" # Skip external-service tests +pytest tests/domain/ # Run specific test directory +pytest -k "test_segment" # Run by pattern + +# Docker (hot-reload enabled) +docker compose up -d postgres # PostgreSQL with pgvector +python scripts/dev_watch_server.py # Auto-reload server +``` + +### Forbidden Docker Operations (without explicit permission) + +- `docker compose build` / `up` / `down` / `restart` +- `docker stop` / `docker kill` +- Any command that would interrupt the hot-reload server + +--- + +## Quality Commands (Makefile) + +**Always use Makefile targets instead of running tools directly.** + +### Primary Quality Commands + +```bash +make quality # Run ALL quality checks (TS + Rust + Python) +make quality-py # Python: lint + type-check + test-quality +``` + +### Python Quality + +```bash +make lint-py # Basedpyright lint → .hygeine/basedpyright.lint.json +make type-check-py # Basedpyright strict mode +make test-quality-py # pytest tests/quality/ +make lint-fix-py # Auto-fix Ruff + Sourcery issues +``` + +--- + +## Architecture + +``` +src/noteflow/ +├── domain/ # Entities, ports, value objects +│ ├── entities/ # Meeting, Segment, Summary, Annotation, NamedEntity, Integration, Project, Processing, SummarizationTemplate +│ ├── identity/ # User, Workspace, WorkspaceMembership, roles, context +│ ├── auth/ # OIDC discovery, claims, constants +│ ├── ports/ # Repository protocols +│ │ ├── repositories/ +│ │ │ ├── transcript.py # MeetingRepository, SegmentRepository, SummaryRepository +│ │ │ ├── asset.py # AssetRepository +│ │ │ ├── background.py # DiarizationJobRepository +│ │ │ ├── external/ # WebhookRepository, IntegrationRepository, EntityRepository, UsageRepository +│ │ │ └── identity/ # UserRepository, WorkspaceRepository, ProjectRepository, MembershipRepository, SummarizationTemplateRepository +│ │ ├── unit_of_work.py # UnitOfWork protocol (supports_* capability checks) +│ │ ├── async_context.py # Async context utilities +│ │ ├── diarization.py # DiarizationEngine protocol +│ │ ├── ner.py # NEREngine protocol +│ │ └── calendar.py # CalendarProvider protocol +│ ├── webhooks/ # WebhookEventType, WebhookConfig, WebhookDelivery, payloads +│ ├── triggers/ # Trigger, TriggerAction, TriggerSignal, TriggerProvider +│ ├── summarization/# SummarizationProvider protocol +│ ├── rules/ # Business rules registry, models, builtin rules +│ ├── settings/ # Domain settings base +│ ├── constants/ # Field definitions, placeholders +│ ├── utils/ # time.py (utc_now), validation.py +│ ├── errors.py # Domain-specific exceptions +│ └── value_objects.py +├── application/ # Use-cases/services +│ ├── services/ +│ │ ├── meeting/ # MeetingService (CRUD, segments, annotations, summaries, state) +│ │ ├── identity/ # IdentityService (context, workspace, defaults) +│ │ ├── calendar/ # CalendarService (connection, events, oauth, sync) +│ │ ├── summarization/ # SummarizationService, TemplateService, ConsentManager +│ │ ├── project_service/ # CRUD, members, roles, rules, active project +│ │ ├── recovery/ # RecoveryService (meeting, job, audio recovery) +│ │ ├── auth/ # AuthService + helpers (service/types/workflows/constants) +│ │ ├── asr_config/ # ASR config service + types/persistence +│ │ ├── streaming_config/ # Streaming config persistence helpers +│ │ ├── export/ # ExportService +│ │ ├── huggingface/ # HfTokenService +│ │ ├── ner/ # NerService +│ │ ├── retention/ # RetentionService +│ │ ├── triggers/ # TriggerService +│ │ ├── webhooks/ # WebhookService +│ │ └── protocols/ # Shared service protocols +│ └── observability/ # Observability ports (ports/) +├── infrastructure/ # Implementations +│ ├── audio/ # sounddevice capture, ring buffer, VU levels, playback, writer, reader +│ ├── asr/ # faster-whisper engine, VAD segmenter, streaming, DTOs +│ ├── auth/ # OIDC discovery, registry, presets +│ ├── diarization/ # Session, assigner, engine (streaming: diart, offline: pyannote) +│ ├── summarization/# CloudProvider, OllamaProvider, MockProvider, parsing, citation verifier, template renderer +│ ├── triggers/ # Calendar, audio_activity, foreground_app providers +│ ├── persistence/ # SQLAlchemy + asyncpg + pgvector +│ │ ├── database.py # create_async_engine, create_async_session_factory +│ │ ├── models/ # ORM models (core/, identity/, integrations/, entities/, observability/, organization/) +│ │ ├── repositories/ # Repository implementations +│ │ │ ├── meeting_repo.py +│ │ │ ├── segment_repo.py +│ │ │ ├── summary_repo.py +│ │ │ ├── annotation_repo.py +│ │ │ ├── entity_repo.py +│ │ │ ├── webhook_repo.py +│ │ │ ├── preferences_repo.py +│ │ │ ├── asset_repo.py +│ │ │ ├── summarization_template_repo.py +│ │ │ ├── diarization_job/ +│ │ │ ├── integration/ +│ │ │ ├── identity/ +│ │ │ ├── usage_event/ +│ │ │ └── _base/ # BaseRepository with _execute_scalar, _execute_scalars, _add_and_flush +│ │ ├── unit_of_work/ +│ │ ├── memory/ # In-memory implementations +│ │ └── migrations/ # Alembic migrations +│ ├── security/ # Keystore (keyring + AES-GCM), protocols, crypto/ +│ ├── crypto/ # Cryptographic utilities +│ ├── export/ # Markdown, HTML, PDF (WeasyPrint), formatting helpers +│ ├── webhooks/ # WebhookExecutor (delivery, signing, metrics) +│ ├── converters/ # ORM ↔ domain (orm, webhook, ner, calendar, integration, asr) +│ ├── calendar/ # Google/Outlook adapters, OAuth flow +│ ├── auth/ # OIDC registry, discovery, presets +│ ├── ner/ # spaCy NER engine +│ ├── observability/# OpenTelemetry tracing (otel.py), usage event tracking +│ ├── metrics/ # Metric collection utilities +│ ├── logging/ # Log buffer and utilities +│ └── platform/ # Platform-specific code +├── grpc/ # gRPC layer +│ ├── proto/ # noteflow.proto, generated *_pb2.py/*_pb2_grpc.py +│ ├── server/ # Bootstrap, lifecycle, setup, services, types +│ ├── service.py # NoteFlowServicer +│ ├── client.py # Python gRPC client wrapper +│ ├── meeting_store.py +│ ├── stream_state.py +│ ├── interceptors/ # Identity context propagation +│ ├── _mixins/ # Server-side gRPC mixins (see below) +│ └── _client_mixins/# Client-side gRPC mixins +├── cli/ # CLI tools +│ ├── __main__.py # CLI entry point +│ ├── retention.py # Retention management +│ ├── constants.py # CLI constants +│ └── models/ # Model commands (package) +│ ├── _download.py +│ ├── _parser.py +│ ├── _registry.py +│ ├── _status.py +│ └── _types.py +└── config/ # Pydantic settings (NOTEFLOW_ env vars) + ├── settings/ # _main.py, _features.py, _triggers.py, _calendar.py, _loaders.py + └── constants/ +``` + +### gRPC Server Mixins (`grpc/_mixins/`) + +``` +_mixins/ +├── streaming/ # ASR streaming (package) +│ ├── _mixin.py # Main StreamingMixin +│ ├── _session.py # Session management +│ ├── _asr.py # ASR processing +│ ├── _processing/ # Audio processing pipeline +│ │ ├── _audio_ops.py +│ │ ├── _chunk_tracking.py +│ │ ├── _congestion.py +│ │ ├── _constants.py +│ │ ├── _types.py +│ │ └── _vad_processing.py +│ ├── _partials.py # Partial transcript handling +│ ├── _cleanup.py # Resource cleanup +│ └── _types.py +├── diarization/ # Speaker diarization (package) +│ ├── _mixin.py # Main DiarizationMixin +│ ├── _jobs.py # Background job management +│ ├── _refinement.py# Offline refinement +│ ├── _streaming.py # Real-time diarization +│ ├── _speaker.py # Speaker assignment +│ ├── _status.py # Job status tracking +│ └── _types.py +├── summarization/ # Summary generation (package) +│ ├── _generation_mixin.py +│ ├── _templates_mixin.py +│ ├── _consent_mixin.py +│ ├── _template_crud.py +│ ├── _template_resolution.py +│ ├── _summary_generation.py +│ ├── _consent.py +│ └── _context_builders.py +├── meeting/ # Meeting lifecycle (package) +│ ├── meeting_mixin.py +│ ├── _project_scope.py +│ └── _stop_ops.py +├── project/ # Project management (package) +│ ├── _mixin.py +│ ├── _membership.py +│ └── _converters.py +├── oidc/ # OIDC authentication (package) +│ ├── oidc_mixin.py +│ └── _support.py +├── converters/ # Proto ↔ domain converters (package) +│ ├── _domain.py +│ ├── _external.py +│ ├── _timestamps.py +│ ├── _id_parsing.py +│ └── _oidc.py +├── errors/ # gRPC error helpers (package) +│ ├── _abort.py # abort_not_found, abort_invalid_argument +│ ├── _require.py # Requirement checks +│ ├── _fetch.py # Fetch with error handling +│ ├── _parse.py # Parsing helpers +│ └── _constants.py +├── servicer_core/ # Core servicer protocols +├── servicer_other/ # Additional servicer protocols +├── annotation.py # Segment annotations CRUD +├── export.py # Markdown/HTML/PDF export +├── entities.py # Named entity extraction +├── calendar.py # Calendar sync operations +├── webhooks.py # Webhook management +├── preferences.py # User preferences +├── observability.py # Usage tracking, metrics +├── identity.py # User/workspace identity +├── sync.py # State synchronization +├── diarization_job.py# Job status/management +├── protocols.py # ServicerHost protocol +├── _types.py +├── _audio_processing.py +├── _repository_protocols.py +└── _servicer_state.py +``` + +### gRPC Client Mixins (`grpc/_client_mixins/`) + +``` +_client_mixins/ +├── streaming.py # Client streaming operations +├── meeting.py # Meeting CRUD operations +├── diarization.py # Diarization requests +├── export.py # Export requests +├── annotation.py # Annotation operations +├── converters.py # Response converters +└── protocols.py # ClientHost protocol +``` + +--- + +## Database + +PostgreSQL with pgvector extension. Async SQLAlchemy with asyncpg driver. + +```bash +# Alembic migrations +alembic upgrade head +alembic revision --autogenerate -m "description" +``` + +Connection via `NOTEFLOW_DATABASE_URL` env var or settings. + +### ORM Models (`persistence/models/`) + +| Directory | Models | +|-----------|--------| +| `core/` | MeetingModel, SegmentModel, SummaryModel, AnnotationModel, DiarizationJobModel | +| `identity/` | UserModel, WorkspaceModel, WorkspaceMembershipModel, ProjectModel, ProjectMembershipModel, SettingsModel | +| `integrations/` | IntegrationModel, IntegrationSecretModel, CalendarEventModel, MeetingCalendarLinkModel, WebhookConfigModel, WebhookDeliveryModel | +| `entities/` | NamedEntityModel, SpeakerModel | +| `observability/` | UsageEventModel | +| `organization/` | SummarizationTemplateModel, TaskModel, TagModel | + +--- + +## Testing Conventions + +- Test files: `test_*.py`, functions: `test_*` +- Markers: `@pytest.mark.slow` (model loading), `@pytest.mark.integration` (external services) +- Integration tests use testcontainers for PostgreSQL +- Asyncio auto-mode enabled + +### Test Quality Gates (`tests/quality/`) + +**After any non-trivial changes**, run: + +```bash +pytest tests/quality/ +``` + +This suite enforces: + +| Check | Description | +|-------|-------------| +| `test_test_smells.py` | No assertion roulette, no conditional test logic, no loops in tests | +| `test_magic_values.py` | No magic numbers in assignments | +| `test_code_smells.py` | Code quality checks | +| `test_duplicate_code.py` | No duplicate code patterns | +| `test_stale_code.py` | No stale/dead code | +| `test_decentralized_helpers.py` | Helpers consolidated properly | +| `test_unnecessary_wrappers.py` | No unnecessary wrapper functions | +| `test_baseline_self.py` | Baseline validation self-checks | + +### Global Fixtures (`tests/conftest.py`) + +**Do not redefine these fixtures:** + +| Fixture | Description | +|---------|-------------| +| `reset_context_vars` | Reset logging context variables | +| `mock_uow` | Mock Unit of Work | +| `crypto` | Crypto utilities | +| `meetings_dir` | Temporary meetings directory | +| `webhook_config` | Single-event webhook config | +| `webhook_config_all_events` | All-events webhook config | +| `sample_datetime` | Sample datetime | +| `calendar_settings` | Calendar settings | +| `meeting_id` | Sample meeting ID | +| `sample_meeting` | Sample Meeting entity | +| `recording_meeting` | Recording-state Meeting | +| `sample_rate` | Audio sample rate | +| `mock_grpc_context` | Mock gRPC context | +| `mockasr_engine` | Mock ASR engine | +| `mock_optional_extras` | Mock optional extras | +| `mock_oauth_manager` | Mock OAuth manager | +| `memory_servicer` | In-memory servicer | +| `approx_float` | Approximate float comparison | +| `approx_sequence` | Approximate sequence comparison | + +--- + +## Code Reuse (CRITICAL) + +**BEFORE writing ANY new code, you MUST search for existing implementations.** + +This is not optional. Redundant code creates maintenance burden, inconsistency, and bugs. + +### Mandatory Search Process + +1. **Search for existing functions** that do what you need: + ```bash + # Use Serena's symbolic tools first + find_symbol with substring_matching=true + search_for_pattern for broader searches + ``` + +2. **Check related modules** - if you need audio device config, check `device.rs`; if you need preferences, check `preferences.ts` + +3. **Look at imports** in similar files - they reveal available utilities + +4. **Only create new code if:** + - No existing implementation exists + - Existing code cannot be reasonably extended + - You have explicit approval for new abstractions + +### Anti-Patterns (FORBIDDEN) + +| Anti-Pattern | Correct Approach | +|--------------|------------------| +| Creating wrapper functions for existing utilities | Use the existing function directly | +| Duplicating validation logic | Find and reuse existing validators | +| Writing new helpers without searching | Search first, ask if unsure | +| "It's faster to write new code" | Technical debt is never faster | + +--- + +## Code Style + +### Python + +- Python 3.12+, 100-char line length +- 4-space indentation +- Naming: `snake_case` modules/functions, `PascalCase` classes, `UPPER_SNAKE_CASE` constants +- Strict basedpyright (0 errors, 0 warnings, 0 notes required) +- Ruff for linting (E, W, F, I, B, C4, UP, SIM, RUF) +- Module soft limit 500 LoC, hard limit 750 LoC +- Generated `*_pb2.py`, `*_pb2_grpc.py` excluded from lint + +--- + +## Type Safety (Zero Tolerance) + +### Forbidden Patterns (Python) + +| Pattern | Why Blocked | Alternative | +|---------|-------------|-------------| +| `# type: ignore` | Bypasses type safety | Fix the actual type error | +| `# pyright: ignore` | Bypasses type safety | Fix the actual type error | +| `Any` type annotations | Creates type safety holes | Use `Protocol`, `TypeVar`, `TypedDict`, or specific types | +| Magic numbers | Hidden intent | Define `typing.Final` constants | +| Loops in tests | Non-deterministic | Use `@pytest.mark.parametrize` | +| Conditionals in tests | Non-deterministic | Use `@pytest.mark.parametrize` | +| Multiple assertions without messages | Hard to debug | Add assertion messages | + +### Type Resolution Hierarchy + +When facing dynamic types: + +1. **`Protocol`** — For duck typing (structural subtyping) +2. **`TypeVar`** — For generics +3. **`TypedDict`** — For structured dictionaries +4. **`cast()`** — Last resort (with comment explaining why) + +### Validation Requirements + +After any code changes: + +```bash +source .venv/bin/activate && basedpyright src/noteflow/ +``` + +**Expected output:** `0 errors, 0 warnings, 0 notes` + +--- + +## Automated Enforcement (Hookify Rules) + +### Protected Files (Require Explicit Permission) + +| File/Directory | What's Blocked | +|----------------|----------------| +| `Makefile` | All modifications | +| `tests/quality/` (except `baselines.json`) | All modifications | +| `pyproject.toml`, `ruff.toml`, `pyrightconfig.json` | All edits | + +### Quality Gate Requirement + +Before completing any code changes: + +```bash +make quality +``` + +All quality checks must pass. + +### Policy: No Ignoring Pre-existing Issues + +If you encounter lint errors, type errors, or test failures—**even if they existed before your changes**—you must either: + +1. Fix immediately (for simple issues) +2. Add to todo list (for complex issues) +3. Launch a subagent to fix (for parallelizable work) + +### Policy: Never Modify Quality Test Allowlists + +**STRICTLY FORBIDDEN** without explicit user permission: + +1. Adding entries to allowlists/whitelists in quality tests +2. Increasing thresholds +3. Adding exclusion patterns to skip files from quality checks +4. Modifying filter functions to bypass detection +5. **Reading or accessing quality test files to check allowlist contents** + +**When quality tests fail, the correct approach is:** + +1. **Fix the actual code** that triggers the violation +2. If the detection is a false positive, **improve the filter logic** +3. **Never** add arbitrary values to allowlists just to make tests pass +4. **Never** read allowlist files to see "what's allowed" + +--- + +## Proto/gRPC + +Proto definitions: `src/noteflow/grpc/proto/noteflow.proto` + +Regenerate after proto changes: + +```bash +python -m grpc_tools.protoc -I src/noteflow/grpc/proto \ + --python_out=src/noteflow/grpc/proto \ + --grpc_python_out=src/noteflow/grpc/proto \ + src/noteflow/grpc/proto/noteflow.proto +``` + +Then run stub patching: + +```bash +python scripts/patch_grpc_stubs.py +``` + +### Sync Points (High Risk of Breakage) + +When changing proto: + +1. **Python stubs** — Regenerate `*_pb2.py`, `*_pb2_grpc.py` +2. **Server mixins** — Update `src/noteflow/grpc/_mixins/` +3. **Python client** — Update `src/noteflow/grpc/client.py` + +--- + +## Key Subsystems + +### Speaker Diarization + +- **Streaming**: diart for real-time speaker detection +- **Offline**: pyannote.audio for post-meeting refinement +- **gRPC**: `RefineSpeakerDiarization`, `GetDiarizationJobStatus`, `RenameSpeaker` + +### Summarization + +- **Providers**: CloudProvider (Anthropic/OpenAI), OllamaProvider (local), MockProvider (testing) +- **Templates**: Configurable tone, format, verbosity +- **Citation verification**: Links summary claims to transcript evidence +- **Consent**: Cloud providers require explicit user consent + +### Export + +- **Formats**: Markdown, HTML, PDF (via WeasyPrint) +- **Content**: Transcript with timestamps, speaker labels, summary + +### Named Entity Recognition (NER) + +- **Engine**: spaCy with transformer models +- **Categories**: person, company, product, technical, acronym, location, date, other +- **Segment tracking**: Entities link to source `segment_ids` + +### Trigger Detection + +- **Signals**: Calendar proximity, audio activity, foreground app +- **Actions**: IGNORE, NOTIFY, AUTO_START + +### Webhooks + +- **Events**: `meeting.completed`, `summary.generated`, `recording.started`, `recording.stopped` +- **Delivery**: Exponential backoff retries +- **Security**: HMAC-SHA256 signing + +### Authentication + +- **OIDC**: OpenID Connect with discovery +- **Providers**: Configurable via OIDC registry + +--- + +## Feature Flags + +| Flag | Default | Controls | +|------|---------|----------| +| `NOTEFLOW_FEATURE_TEMPLATES_ENABLED` | `true` | AI summarization templates | +| `NOTEFLOW_FEATURE_PDF_EXPORT_ENABLED` | `true` | PDF export format | +| `NOTEFLOW_FEATURE_NER_ENABLED` | `false` | Named entity extraction | +| `NOTEFLOW_FEATURE_CALENDAR_ENABLED` | `false` | Calendar sync | +| `NOTEFLOW_FEATURE_WEBHOOKS_ENABLED` | `true` | Webhook notifications | + +Access via `get_feature_flags().` or `get_settings().feature_flags.`. + +--- + +## Common Pitfalls Checklist + +### When Adding New Features + +- [ ] Update proto schema first (if gRPC involved) +- [ ] Regenerate Python stubs +- [ ] Run `scripts/patch_grpc_stubs.py` +- [ ] Implement server mixin +- [ ] Update Python client wrapper +- [ ] Add tests (both backend and client) +- [ ] Run `make quality` + +### When Changing Database Schema + +- [ ] Update ORM models in `persistence/models/` +- [ ] Create Alembic migration +- [ ] Update repository implementation +- [ ] Update UnitOfWork if needed +- [ ] Update converters in `infrastructure/converters/` + +### When Modifying Existing Code + +- [ ] Search for all usages first +- [ ] Update all call sites +- [ ] Run `make quality` +- [ ] Run relevant tests + +--- + +## Known Issues & Technical Debt + +See `docs/triage.md` for tracked technical debt. +See `docs/sprints/` for feature implementation plans. diff --git a/src/noteflow/application/observability/ports.py b/src/noteflow/application/observability/ports/__init__.py similarity index 100% rename from src/noteflow/application/observability/ports.py rename to src/noteflow/application/observability/ports/__init__.py diff --git a/src/noteflow/application/services/__init__.py b/src/noteflow/application/services/__init__.py index dfb6d1f..a491c52 100644 --- a/src/noteflow/application/services/__init__.py +++ b/src/noteflow/application/services/__init__.py @@ -1,19 +1,18 @@ """Application services for NoteFlow use cases.""" -from noteflow.application.services.auth_service import ( +from noteflow.application.services.auth import ( AuthResult, AuthService, AuthServiceError, LogoutResult, UserInfo, ) -from noteflow.application.services.export_service import ExportService -from noteflow.domain.value_objects import ExportFormat +from noteflow.application.services.export import ExportFormat, ExportService from noteflow.application.services.identity import IdentityService from noteflow.application.services.meeting import MeetingService from noteflow.application.services.project_service import ProjectService from noteflow.application.services.recovery import RecoveryService -from noteflow.application.services.retention_service import RetentionReport, RetentionService +from noteflow.application.services.retention import RetentionReport, RetentionService from noteflow.application.services.summarization import ( SummarizationMode, SummarizationService, @@ -22,7 +21,7 @@ from noteflow.application.services.summarization import ( SummarizationTemplateService, TemplateUpdateResult, ) -from noteflow.application.services.trigger_service import TriggerService, TriggerServiceSettings +from noteflow.application.services.triggers import TriggerService, TriggerServiceSettings __all__ = [ "AuthResult", diff --git a/src/noteflow/application/services/asr_config/__init__.py b/src/noteflow/application/services/asr_config/__init__.py new file mode 100644 index 0000000..b294760 --- /dev/null +++ b/src/noteflow/application/services/asr_config/__init__.py @@ -0,0 +1,31 @@ +"""ASR configuration services and helpers.""" + +from .persistence import ( + AsrConfigPreference, + AsrPreferenceResolution, + build_asr_config_preference, + resolve_asr_config_preference, +) +from .service import AsrConfigService +from .types import ( + DEVICE_COMPUTE_TYPES, + AsrCapabilities, + AsrComputeType, + AsrConfigJob, + AsrConfigPhase, + AsrDevice, +) + +__all__ = [ + "AsrCapabilities", + "AsrComputeType", + "AsrConfigJob", + "AsrConfigPhase", + "AsrConfigPreference", + "AsrConfigService", + "AsrDevice", + "AsrPreferenceResolution", + "DEVICE_COMPUTE_TYPES", + "build_asr_config_preference", + "resolve_asr_config_preference", +] diff --git a/src/noteflow/application/services/asr_config_persistence.py b/src/noteflow/application/services/asr_config/persistence.py similarity index 95% rename from src/noteflow/application/services/asr_config_persistence.py rename to src/noteflow/application/services/asr_config/persistence.py index 0d654e8..e13a468 100644 --- a/src/noteflow/application/services/asr_config_persistence.py +++ b/src/noteflow/application/services/asr_config/persistence.py @@ -5,11 +5,7 @@ from __future__ import annotations from dataclasses import dataclass from typing import Final, Protocol, TypedDict, cast -from noteflow.application.services.asr_config_types import ( - AsrComputeType, - AsrDevice, - DEVICE_COMPUTE_TYPES, -) +from .types import AsrComputeType, AsrDevice, DEVICE_COMPUTE_TYPES from noteflow.domain.constants.fields import DEVICE from noteflow.infrastructure.asr import VALID_MODEL_SIZES from noteflow.infrastructure.logging import get_logger @@ -118,7 +114,7 @@ def _parse_preference(raw_value: object) -> AsrConfigPreference | None: if compute_type is not None: preference[_PREF_COMPUTE_KEY] = compute_type - return preference if preference else None + return preference or None def _read_string(value: object) -> str | None: @@ -131,9 +127,7 @@ def _read_string(value: object) -> str | None: def _resolve_model_size(preferred: str | None, fallback: str) -> str: if preferred in VALID_MODEL_SIZES: return preferred - if fallback in VALID_MODEL_SIZES: - return fallback - return VALID_MODEL_SIZES[0] + return fallback if fallback in VALID_MODEL_SIZES else VALID_MODEL_SIZES[0] def _resolve_device( diff --git a/src/noteflow/application/services/asr_config_service.py b/src/noteflow/application/services/asr_config/service.py similarity index 98% rename from src/noteflow/application/services/asr_config_service.py rename to src/noteflow/application/services/asr_config/service.py index 2baa061..a43f355 100644 --- a/src/noteflow/application/services/asr_config_service.py +++ b/src/noteflow/application/services/asr_config/service.py @@ -5,7 +5,7 @@ from collections.abc import Awaitable, Callable from typing import TYPE_CHECKING from uuid import UUID, uuid4 -from noteflow.application.services.asr_config_types import ( +from .types import ( DEVICE_COMPUTE_TYPES, AsrCapabilities, AsrComputeType, @@ -335,5 +335,5 @@ class AsrConfigService: except TimeoutError: logger.warning( "asr_config_shutdown_timeout", - remaining_jobs=sum(1 for t in tasks_to_cancel if not t.done()), + remaining_jobs=sum(not t.done() for t in tasks_to_cancel), ) diff --git a/src/noteflow/application/services/asr_config_types.py b/src/noteflow/application/services/asr_config/types.py similarity index 100% rename from src/noteflow/application/services/asr_config_types.py rename to src/noteflow/application/services/asr_config/types.py diff --git a/src/noteflow/application/services/auth/__init__.py b/src/noteflow/application/services/auth/__init__.py new file mode 100644 index 0000000..e91a7a2 --- /dev/null +++ b/src/noteflow/application/services/auth/__init__.py @@ -0,0 +1,14 @@ +"""Authentication application services.""" + +from .constants import DEFAULT_USER_ID, DEFAULT_WORKSPACE_ID +from .service import AuthResult, AuthService, AuthServiceError, LogoutResult, UserInfo + +__all__ = [ + "DEFAULT_USER_ID", + "DEFAULT_WORKSPACE_ID", + "AuthResult", + "AuthService", + "AuthServiceError", + "LogoutResult", + "UserInfo", +] diff --git a/src/noteflow/application/services/auth_constants.py b/src/noteflow/application/services/auth/constants.py similarity index 100% rename from src/noteflow/application/services/auth_constants.py rename to src/noteflow/application/services/auth/constants.py diff --git a/src/noteflow/application/services/auth_integration_manager.py b/src/noteflow/application/services/auth/integration_manager.py similarity index 98% rename from src/noteflow/application/services/auth_integration_manager.py rename to src/noteflow/application/services/auth/integration_manager.py index 5305559..1d31a56 100644 --- a/src/noteflow/application/services/auth_integration_manager.py +++ b/src/noteflow/application/services/auth/integration_manager.py @@ -8,7 +8,7 @@ from uuid import UUID from noteflow.domain.entities.integration import IntegrationType -from .auth_workflows import ( +from .workflows import ( AuthIntegrationContext, get_or_create_auth_integration, get_or_create_default_workspace_id, diff --git a/src/noteflow/application/services/auth_service.py b/src/noteflow/application/services/auth/service.py similarity index 95% rename from src/noteflow/application/services/auth_service.py rename to src/noteflow/application/services/auth/service.py index be068dc..4809c5c 100644 --- a/src/noteflow/application/services/auth_service.py +++ b/src/noteflow/application/services/auth/service.py @@ -16,11 +16,11 @@ from noteflow.infrastructure.calendar import OAuthManager from noteflow.infrastructure.calendar.oauth import OAuthError from noteflow.infrastructure.logging import get_logger -from .auth_constants import DEFAULT_USER_ID, DEFAULT_WORKSPACE_ID -from .auth_integration_manager import IntegrationManager -from .auth_token_exchanger import AuthServiceError, TokenExchanger -from .auth_types import AuthResult, LogoutResult, UserInfo -from .auth_workflows import find_connected_auth_integration, resolve_display_name +from .constants import DEFAULT_USER_ID, DEFAULT_WORKSPACE_ID +from .integration_manager import IntegrationManager +from .token_exchanger import AuthServiceError, TokenExchanger +from .types import AuthResult, LogoutResult, UserInfo +from .workflows import find_connected_auth_integration, resolve_display_name @dataclass(frozen=True) @@ -289,9 +289,10 @@ class AuthService: if integration is None or not integration.is_connected: return None - return await self._token_exchanger.refresh_tokens( + result = await self._token_exchanger.refresh_tokens( uow, oauth_provider, integration ) + return result.auth_result if result.success else None @staticmethod def _parse_auth_provider(provider: str) -> OAuthProvider: diff --git a/src/noteflow/application/services/auth_token_exchanger.py b/src/noteflow/application/services/auth/token_exchanger.py similarity index 80% rename from src/noteflow/application/services/auth_token_exchanger.py rename to src/noteflow/application/services/auth/token_exchanger.py index 33fbcc4..d158318 100644 --- a/src/noteflow/application/services/auth_token_exchanger.py +++ b/src/noteflow/application/services/auth/token_exchanger.py @@ -6,6 +6,7 @@ from typing import TYPE_CHECKING from uuid import UUID from noteflow.config.constants import OAUTH_FIELD_ACCESS_TOKEN +from noteflow.config.constants.errors import ERR_TOKEN_REFRESH_PREFIX from noteflow.domain.value_objects import OAuthProvider, OAuthTokens from noteflow.infrastructure.calendar import OAuthManager from noteflow.infrastructure.calendar.google_adapter import GoogleCalendarError @@ -13,8 +14,8 @@ from noteflow.infrastructure.calendar.oauth import OAuthError from noteflow.infrastructure.calendar.outlook import OutlookCalendarError from noteflow.infrastructure.logging import get_logger -from .auth_types import AuthResult -from .auth_workflows import refresh_tokens_for_integration +from .types import TokenRefreshResult +from .workflows import refresh_tokens_for_integration if TYPE_CHECKING: from noteflow.domain.entities.integration import Integration @@ -128,17 +129,36 @@ class TokenExchanger: uow: UnitOfWork, oauth_provider: OAuthProvider, integration: Integration, - ) -> AuthResult | None: + ) -> TokenRefreshResult: """Refresh expired auth tokens.""" try: - return await refresh_tokens_for_integration( + result = await refresh_tokens_for_integration( uow, oauth_provider=oauth_provider, integration=integration, oauth_manager=self._oauth_manager, ) + if not result.success: + integration.mark_error(f"{ERR_TOKEN_REFRESH_PREFIX}{result.error}") + await uow.integrations.update(integration) + await uow.commit() + return TokenRefreshResult( + error=result.error, + integration_marked_error=True, + ) + return result except OAuthError as e: - integration.mark_error(f"Token refresh failed: {e}") + error_msg = f"{ERR_TOKEN_REFRESH_PREFIX}{e}" + integration.mark_error(error_msg) await uow.integrations.update(integration) await uow.commit() - return None + logger.warning( + "token_refresh_oauth_error", + integration_id=str(integration.id), + provider=oauth_provider.value, + error=error_msg, + ) + return TokenRefreshResult( + error=error_msg, + integration_marked_error=True, + ) diff --git a/src/noteflow/application/services/auth_types.py b/src/noteflow/application/services/auth/types.py similarity index 86% rename from src/noteflow/application/services/auth_types.py rename to src/noteflow/application/services/auth/types.py index 5849e93..fe14289 100644 --- a/src/noteflow/application/services/auth_types.py +++ b/src/noteflow/application/services/auth/types.py @@ -5,6 +5,8 @@ from __future__ import annotations from dataclasses import dataclass from uuid import UUID +from noteflow.domain.value_objects import OAuthTokens + @dataclass(frozen=True, slots=True) class AuthResult: @@ -102,3 +104,18 @@ class LogoutResult: tokens_revoked=tokens_revoked, revocation_error=error, ) + + +@dataclass(frozen=True, slots=True) +class TokenRefreshResult: + """Result of token refresh operation.""" + + tokens: OAuthTokens | None = None + auth_result: AuthResult | None = None + error: str | None = None + integration_marked_error: bool = False + + @property + def success(self) -> bool: + """Check if token refresh was successful.""" + return self.auth_result is not None and self.error is None diff --git a/src/noteflow/application/services/auth_workflows.py b/src/noteflow/application/services/auth/workflows.py similarity index 89% rename from src/noteflow/application/services/auth_workflows.py rename to src/noteflow/application/services/auth/workflows.py index 2ea8c90..a2fa8dc 100644 --- a/src/noteflow/application/services/auth_workflows.py +++ b/src/noteflow/application/services/auth/workflows.py @@ -13,8 +13,8 @@ from noteflow.domain.value_objects import OAuthProvider, OAuthTokens from noteflow.infrastructure.calendar import OAuthManager from noteflow.infrastructure.logging import get_logger -from .auth_constants import DEFAULT_USER_ID, DEFAULT_WORKSPACE_ID -from .auth_types import AuthResult +from .constants import DEFAULT_USER_ID, DEFAULT_WORKSPACE_ID +from .types import AuthResult, TokenRefreshResult if TYPE_CHECKING: from noteflow.domain.ports.unit_of_work import UnitOfWork @@ -177,19 +177,24 @@ async def refresh_tokens_for_integration( oauth_provider: OAuthProvider, integration: Integration, oauth_manager: OAuthManager, -) -> AuthResult | None: +) -> TokenRefreshResult: """Refresh tokens for a connected integration if needed.""" secrets = await uow.integrations.get_secrets(integration.id) if not secrets: - return None + return TokenRefreshResult(error="No secrets found for integration") try: tokens = OAuthTokens.from_secrets_dict(secrets) - except (KeyError, ValueError): - return None + except (KeyError, ValueError) as e: + logger.warning( + "token_parse_failed", + error_type=type(e).__name__, + integration_id=str(integration.id), + ) + return TokenRefreshResult(error=f"Invalid token format: {e}") if not tokens.refresh_token: - return None + return TokenRefreshResult(error="No refresh token available") if not tokens.is_expired(buffer_seconds=TOKEN_EXPIRY_BUFFER_SECONDS): logger.debug( @@ -198,12 +203,13 @@ async def refresh_tokens_for_integration( expires_at=tokens.expires_at.isoformat() if tokens.expires_at else None, ) user_id = resolve_user_id_from_integration(integration) - return AuthResult( + auth_result = AuthResult( user_id=user_id, workspace_id=DEFAULT_WORKSPACE_ID, display_name=resolve_provider_email(integration), email=integration.provider_email, ) + return TokenRefreshResult(tokens=tokens, auth_result=auth_result) new_tokens = await oauth_manager.refresh_tokens( provider=oauth_provider, @@ -214,9 +220,10 @@ async def refresh_tokens_for_integration( await uow.commit() user_id = resolve_user_id_from_integration(integration) - return AuthResult( + auth_result = AuthResult( user_id=user_id, workspace_id=DEFAULT_WORKSPACE_ID, display_name=resolve_provider_email(integration), email=integration.provider_email, ) + return TokenRefreshResult(tokens=new_tokens, auth_result=auth_result) diff --git a/src/noteflow/application/services/export/__init__.py b/src/noteflow/application/services/export/__init__.py new file mode 100644 index 0000000..93ba822 --- /dev/null +++ b/src/noteflow/application/services/export/__init__.py @@ -0,0 +1,10 @@ +"""Export application services.""" + +from noteflow.domain.value_objects import ExportFormat + +from .service import ExportService + +__all__ = [ + "ExportFormat", + "ExportService", +] diff --git a/src/noteflow/application/services/export_service.py b/src/noteflow/application/services/export/service.py similarity index 99% rename from src/noteflow/application/services/export_service.py rename to src/noteflow/application/services/export/service.py index 8a22150..e6c080d 100644 --- a/src/noteflow/application/services/export_service.py +++ b/src/noteflow/application/services/export/service.py @@ -22,7 +22,7 @@ from noteflow.infrastructure.export import ( ) from noteflow.infrastructure.logging import get_logger -from .protocols import ExportRepositoryProvider +from noteflow.application.services.protocols import ExportRepositoryProvider if TYPE_CHECKING: from noteflow.domain.entities import Meeting, Segment diff --git a/src/noteflow/application/services/huggingface/__init__.py b/src/noteflow/application/services/huggingface/__init__.py new file mode 100644 index 0000000..55d32c3 --- /dev/null +++ b/src/noteflow/application/services/huggingface/__init__.py @@ -0,0 +1,9 @@ +"""HuggingFace integration services.""" + +from .service import HfTokenService, HfTokenStatus, HfValidationResult + +__all__ = [ + "HfTokenService", + "HfTokenStatus", + "HfValidationResult", +] diff --git a/src/noteflow/application/services/hf_token_service.py b/src/noteflow/application/services/huggingface/service.py similarity index 100% rename from src/noteflow/application/services/hf_token_service.py rename to src/noteflow/application/services/huggingface/service.py diff --git a/src/noteflow/application/services/identity/identity_service.py b/src/noteflow/application/services/identity/identity_service.py index 986b9c5..47b36a2 100644 --- a/src/noteflow/application/services/identity/identity_service.py +++ b/src/noteflow/application/services/identity/identity_service.py @@ -10,7 +10,7 @@ from __future__ import annotations from typing import TYPE_CHECKING from uuid import UUID, uuid4 -from noteflow.domain.constants.fields import EMAIL +from noteflow.domain.constants.fields import DISPLAY_NAME, EMAIL from noteflow.domain.entities.project import slugify from noteflow.domain.identity import ( DEFAULT_PROJECT_NAME, @@ -219,7 +219,7 @@ class IdentityService( updated_fields: list[str] = [] if display_name: user.display_name = display_name - updated_fields.append("display_name") + updated_fields.append(DISPLAY_NAME) if email is not None: user.email = email updated_fields.append(EMAIL) diff --git a/src/noteflow/application/services/ner/__init__.py b/src/noteflow/application/services/ner/__init__.py new file mode 100644 index 0000000..da635cd --- /dev/null +++ b/src/noteflow/application/services/ner/__init__.py @@ -0,0 +1,8 @@ +"""Named entity extraction services.""" + +from .service import ExtractionResult, NerService + +__all__ = [ + "ExtractionResult", + "NerService", +] diff --git a/src/noteflow/application/services/ner_service.py b/src/noteflow/application/services/ner/service.py similarity index 100% rename from src/noteflow/application/services/ner_service.py rename to src/noteflow/application/services/ner/service.py diff --git a/src/noteflow/application/services/protocols.py b/src/noteflow/application/services/protocols/__init__.py similarity index 91% rename from src/noteflow/application/services/protocols.py rename to src/noteflow/application/services/protocols/__init__.py index 52cc2c4..744cc7c 100644 --- a/src/noteflow/application/services/protocols.py +++ b/src/noteflow/application/services/protocols/__init__.py @@ -15,3 +15,8 @@ class ExportRepositoryProvider(AsyncContextManager, Protocol): meetings: MeetingRepository segments: SegmentRepository + + +__all__ = [ + "ExportRepositoryProvider", +] diff --git a/src/noteflow/application/services/recovery/_job_recoverer.py b/src/noteflow/application/services/recovery/_job_recoverer.py index 0c9fdb2..8186e0c 100644 --- a/src/noteflow/application/services/recovery/_job_recoverer.py +++ b/src/noteflow/application/services/recovery/_job_recoverer.py @@ -2,6 +2,7 @@ from __future__ import annotations +from dataclasses import dataclass from typing import TYPE_CHECKING import sqlalchemy.exc @@ -14,6 +15,20 @@ if TYPE_CHECKING: logger = get_logger(__name__) +@dataclass(frozen=True, slots=True) +class JobRecoveryResult: + """Result of job recovery operation.""" + + jobs_recovered: int = 0 + migration_required: bool = False + error: str | None = None + + @property + def success(self) -> bool: + """Check if recovery was successful.""" + return not self.migration_required and self.error is None + + class DiarizationJobRecoverer: """Handle recovery of crashed diarization jobs.""" @@ -25,16 +40,26 @@ class DiarizationJobRecoverer: """ self._uow = uow - async def recover(self) -> int: + async def recover(self) -> JobRecoveryResult: """Mark diarization jobs left in running states as failed. Returns: - Number of jobs marked as failed. + Result of recovery operation. """ try: - return await self._mark_jobs_failed() + count = await self._mark_jobs_failed() + return JobRecoveryResult(jobs_recovered=count) except sqlalchemy.exc.ProgrammingError as e: - return handle_missing_diarization_table(e) + if "does not exist" in str(e) or "UndefinedTableError" in str(e): + logger.warning( + "recovery_migration_required", + error=str(e), + ) + return JobRecoveryResult( + migration_required=True, + error="Diarization table missing - migration required", + ) + raise async def _mark_jobs_failed(self) -> int: """Mark running diarization jobs as failed and log result.""" @@ -55,14 +80,3 @@ def log_diarization_recovery(failed_count: int) -> None: ) else: logger.info("No crashed diarization jobs found during recovery") - - -def handle_missing_diarization_table(error: sqlalchemy.exc.ProgrammingError) -> int: - """Handle case where diarization_jobs table doesn't exist yet.""" - if "does not exist" in str(error) or "UndefinedTableError" in str(error): - logger.debug( - "Diarization jobs table not found during recovery, skipping: %s", - error, - ) - return 0 - raise error diff --git a/src/noteflow/application/services/recovery/recovery_service.py b/src/noteflow/application/services/recovery/recovery_service.py index c8de8ca..5e3937c 100644 --- a/src/noteflow/application/services/recovery/recovery_service.py +++ b/src/noteflow/application/services/recovery/recovery_service.py @@ -15,7 +15,7 @@ from noteflow.infrastructure.logging import get_logger from noteflow.infrastructure.persistence.constants import MAX_MEETINGS_LIMIT from ._audio_validator import AudioValidationResult, AudioValidator -from ._job_recoverer import DiarizationJobRecoverer +from ._job_recoverer import DiarizationJobRecoverer, JobRecoveryResult from ._meeting_recoverer import MeetingRecoverer if TYPE_CHECKING: @@ -159,11 +159,11 @@ class RecoveryService: total += await self._uow.meetings.count_by_state(state) return total - async def recover_crashed_diarization_jobs(self) -> int: + async def recover_crashed_diarization_jobs(self) -> JobRecoveryResult: """Mark diarization jobs left in running states as failed. Returns: - Number of jobs marked as failed. + Result of job recovery operation. """ return await self._job_recoverer.recover() @@ -177,11 +177,11 @@ class RecoveryService: RecoveryResult with counts of recovered items. """ meetings, audio_failures = await self.recover_crashed_meetings() - jobs = await self.recover_crashed_diarization_jobs() + job_result = await self.recover_crashed_diarization_jobs() result = RecoveryResult( meetings_recovered=len(meetings), - diarization_jobs_failed=jobs, + diarization_jobs_failed=job_result.jobs_recovered if job_result.success else 0, audio_validation_failures=audio_failures, ) @@ -192,6 +192,13 @@ class RecoveryService: result.diarization_jobs_failed, ) + if not job_result.success: + logger.warning( + "Diarization job recovery incomplete", + migration_required=job_result.migration_required, + error=job_result.error, + ) + return result diff --git a/src/noteflow/application/services/retention/__init__.py b/src/noteflow/application/services/retention/__init__.py new file mode 100644 index 0000000..b4939f7 --- /dev/null +++ b/src/noteflow/application/services/retention/__init__.py @@ -0,0 +1,8 @@ +"""Retention services.""" + +from .service import RetentionReport, RetentionService + +__all__ = [ + "RetentionReport", + "RetentionService", +] diff --git a/src/noteflow/application/services/retention_service.py b/src/noteflow/application/services/retention/service.py similarity index 79% rename from src/noteflow/application/services/retention_service.py rename to src/noteflow/application/services/retention/service.py index aa7ea30..7b2f7e2 100644 --- a/src/noteflow/application/services/retention_service.py +++ b/src/noteflow/application/services/retention/service.py @@ -16,6 +16,20 @@ if TYPE_CHECKING: logger = get_logger(__name__) +@dataclass(frozen=True, slots=True) +class DeletionResult: + """Result of meeting deletion operation.""" + + meeting_id: str + deleted: bool = False + error: str | None = None + + @property + def success(self) -> bool: + """Check if deletion was successful.""" + return self.deleted and self.error is None + + @dataclass(frozen=True) class RetentionReport: """Result of retention cleanup run. @@ -137,10 +151,11 @@ class RetentionService: for meeting in meetings: result = await self._try_delete_meeting(meeting, MeetingService) - if result is None: + if result.success: deleted += 1 else: - errors.append(result) + error_msg = f"{result.meeting_id}: {result.error}" + errors.append(error_msg) return deleted, errors @@ -148,18 +163,31 @@ class RetentionService: self, meeting: Meeting, meeting_service_cls: type, - ) -> str | None: - """Attempt to delete a single meeting. Returns error message or None on success.""" + ) -> DeletionResult: + """Attempt to delete a single meeting.""" + meeting_id = str(meeting.id) try: meeting_svc = meeting_service_cls(self._uow_factory()) success = await meeting_svc.delete_meeting(meeting.id) if success: - logger.info("Deleted expired meeting: id=%s", meeting.id) - return None - return f"{meeting.id}: deletion returned False" + logger.info( + "retention_meeting_deleted", + meeting_id=meeting_id, + ) + return DeletionResult(meeting_id=meeting_id, deleted=True) + logger.warning( + "retention_deletion_returned_false", + meeting_id=meeting_id, + ) + return DeletionResult(meeting_id=meeting_id, error="deletion returned False") except (OSError, RuntimeError) as e: - logger.warning("Failed to delete meeting %s: %s", meeting.id, e) - return f"{meeting.id}: {e}" + logger.warning( + "retention_deletion_failed", + meeting_id=meeting_id, + error_type=type(e).__name__, + error=str(e), + ) + return DeletionResult(meeting_id=meeting_id, error=str(e)) @staticmethod def _log_cleanup_complete(checked: int, deleted: int, error_count: int) -> None: diff --git a/src/noteflow/application/services/streaming_config/__init__.py b/src/noteflow/application/services/streaming_config/__init__.py new file mode 100644 index 0000000..987cbad --- /dev/null +++ b/src/noteflow/application/services/streaming_config/__init__.py @@ -0,0 +1,23 @@ +"""Streaming configuration helpers.""" + +from .persistence import ( + STREAMING_CONFIG_KEYS, + STREAMING_CONFIG_RANGES, + StreamingConfig, + StreamingConfigPreference, + StreamingConfigResolution, + build_default_streaming_config, + build_streaming_config_preference, + resolve_streaming_config_preference, +) + +__all__ = [ + "STREAMING_CONFIG_KEYS", + "STREAMING_CONFIG_RANGES", + "StreamingConfig", + "StreamingConfigPreference", + "StreamingConfigResolution", + "build_default_streaming_config", + "build_streaming_config_preference", + "resolve_streaming_config_preference", +] diff --git a/src/noteflow/application/services/streaming_config_persistence.py b/src/noteflow/application/services/streaming_config/persistence.py similarity index 97% rename from src/noteflow/application/services/streaming_config_persistence.py rename to src/noteflow/application/services/streaming_config/persistence.py index 2a38dbc..141fcf4 100644 --- a/src/noteflow/application/services/streaming_config_persistence.py +++ b/src/noteflow/application/services/streaming_config/persistence.py @@ -132,7 +132,7 @@ def _parse_preference(raw_value: object) -> StreamingConfigPreference | None: if value is not None: preference[key] = value - return preference if preference else None + return preference or None def _resolve_config( @@ -187,6 +187,4 @@ def _clamp(value: float, min_val: float, max_val: float) -> float: def _read_float(value: object) -> float | None: - if isinstance(value, (float, int)): - return float(value) - return None + return float(value) if isinstance(value, (float, int)) else None diff --git a/src/noteflow/application/services/triggers/__init__.py b/src/noteflow/application/services/triggers/__init__.py new file mode 100644 index 0000000..5bea647 --- /dev/null +++ b/src/noteflow/application/services/triggers/__init__.py @@ -0,0 +1,8 @@ +"""Trigger services.""" + +from .service import TriggerService, TriggerServiceSettings + +__all__ = [ + "TriggerService", + "TriggerServiceSettings", +] diff --git a/src/noteflow/application/services/trigger_service.py b/src/noteflow/application/services/triggers/service.py similarity index 100% rename from src/noteflow/application/services/trigger_service.py rename to src/noteflow/application/services/triggers/service.py diff --git a/src/noteflow/application/services/webhooks/__init__.py b/src/noteflow/application/services/webhooks/__init__.py new file mode 100644 index 0000000..371f796 --- /dev/null +++ b/src/noteflow/application/services/webhooks/__init__.py @@ -0,0 +1,7 @@ +"""Webhook services.""" + +from .service import WebhookService + +__all__ = [ + "WebhookService", +] diff --git a/src/noteflow/application/services/webhook_service.py b/src/noteflow/application/services/webhooks/service.py similarity index 100% rename from src/noteflow/application/services/webhook_service.py rename to src/noteflow/application/services/webhooks/service.py diff --git a/src/noteflow/config/constants/core.py b/src/noteflow/config/constants/core.py index 67c3e86..68613ec 100644 --- a/src/noteflow/config/constants/core.py +++ b/src/noteflow/config/constants/core.py @@ -44,6 +44,9 @@ DEFAULT_LLM_TEMPERATURE: Final[float] = 0.3 DEFAULT_OLLAMA_TIMEOUT_SECONDS: Final[float] = float(60 * 2) """Default timeout for Ollama requests in seconds.""" +DEFAULT_OLLAMA_HOST: Final[str] = "http://localhost:11434" +"""Default Ollama server host URL.""" + # ============================================================================= # gRPC Settings # ============================================================================= diff --git a/src/noteflow/config/settings/_main.py b/src/noteflow/config/settings/_main.py index 69b2044..ec4f587 100644 --- a/src/noteflow/config/settings/_main.py +++ b/src/noteflow/config/settings/_main.py @@ -10,6 +10,7 @@ from noteflow.config.constants import APP_DIR_NAME from noteflow.config.constants.core import ( DAYS_PER_WEEK, DEFAULT_LLM_TEMPERATURE, + DEFAULT_OLLAMA_HOST, DEFAULT_OLLAMA_TIMEOUT_SECONDS, HOURS_PER_DAY, ) @@ -251,7 +252,7 @@ class Settings(TriggerSettings): # Ollama settings ollama_host: Annotated[ str, - Field(default="http://localhost:11434", description="Ollama server host URL"), + Field(default=DEFAULT_OLLAMA_HOST, description="Ollama server host URL"), ] ollama_timeout_seconds: Annotated[ float, diff --git a/src/noteflow/domain/constants/fields.py b/src/noteflow/domain/constants/fields.py index c22c0cf..d55a94a 100644 --- a/src/noteflow/domain/constants/fields.py +++ b/src/noteflow/domain/constants/fields.py @@ -3,6 +3,7 @@ from typing import Final, Literal EMAIL: Final[str] = "email" +DISPLAY_NAME: Final[Literal["display_name"]] = "display_name" GROUPS: Final[str] = "groups" PROFILE: Final[str] = "profile" ENABLED: Final[str] = "enabled" @@ -22,6 +23,8 @@ PROVIDER_NAME: Final[Literal["provider_name"]] = "provider_name" NOTE: Final[str] = "note" START_TIME: Final[Literal["start_time"]] = "start_time" END_TIME: Final[Literal["end_time"]] = "end_time" +STATE: Final[Literal["state"]] = "state" +ENDED_AT: Final[Literal["ended_at"]] = "ended_at" CODE: Final[str] = "code" CONTENT: Final[str] = "content" LOCATION: Final[str] = "location" diff --git a/src/noteflow/domain/entities/meeting.py b/src/noteflow/domain/entities/meeting.py index 2db7246..757ea35 100644 --- a/src/noteflow/domain/entities/meeting.py +++ b/src/noteflow/domain/entities/meeting.py @@ -7,7 +7,7 @@ from datetime import datetime from typing import TYPE_CHECKING from uuid import UUID, uuid4 -from noteflow.domain.constants.fields import ASSET_PATH, PROJECT_ID, WRAPPED_DEK +from noteflow.domain.constants.fields import ASSET_PATH, ENDED_AT, PROJECT_ID, STATE, WRAPPED_DEK from noteflow.domain.entities.processing import ( ProcessingStatus, ProcessingStepState, @@ -90,10 +90,10 @@ class MeetingLoadParams: PROJECT_ID: self.project_id, "workspace_id": self.workspace_id or DEFAULT_WORKSPACE_ID, "created_by_id": self.created_by_id or DEFAULT_USER_ID, - "state": self.state, + STATE: self.state, "created_at": self.created_at or utc_now(), "started_at": self.started_at, - "ended_at": self.ended_at, + ENDED_AT: self.ended_at, "metadata": self.metadata or {}, WRAPPED_DEK: self.wrapped_dek, ASSET_PATH: self.asset_path or fallback_asset_path, diff --git a/src/noteflow/grpc/AGENTS.md b/src/noteflow/grpc/AGENTS.md new file mode 100644 index 0000000..0e97023 --- /dev/null +++ b/src/noteflow/grpc/AGENTS.md @@ -0,0 +1,638 @@ +# gRPC Security Rules + +Security rules for gRPC development in NoteFlow. + +## Prerequisites + +- `../../.agent/rules/_core/owasp-2025.md` - Core web security +- `../../.agent/rules/languages/python/CLAUDE.md` - Python security +- NoteFlow gRPC schema: `proto/noteflow.proto` - Shared contract + +--- + +## Input Validation + +### Rule: Use Protobuf Message Validation + +**Level**: `strict` + +**When**: Accepting request data from clients. + +**Do**: +```python +import grpc +from pydantic import BaseModel, Field, validator +from noteflow_pb2 import CreateUserRequest, UserResponse + +class PydanticUserCreate(BaseModel): + email: str = Field(..., regex=r'^[^@]+@[^@]+\.[^@]+$') + password: str = Field(..., min_length=8, max_length=128) + age: int = Field(..., ge=0, le=150) + + @validator('password') + def password_strength(cls, v): + if not any(c.isupper() for c in v): + raise ValueError('Password must contain uppercase') + if not any(c.isdigit() for c in v): + raise ValueError('Password must contain digit') + return v + +class UserServicer(noteflow_pb2_grpc.UserServiceServicer): + def CreateUser(self, request: CreateUserRequest, context: grpc.ServicerContext) -> UserResponse: + # Validate with Pydantic before processing + try: + validated_data = PydanticUserCreate( + email=request.email, + password=request.password, + age=request.age + ) + except ValueError as e: + context.set_code(grpc.StatusCode.INVALID_ARGUMENT) + context.set_details(str(e)) + return UserResponse() + + # Process validated data + return create_user_in_db(validated_data.dict()) +``` + +**Don't**: +```python +class UserServicer(noteflow_pb2_grpc.UserServiceServicer): + def CreateUser(self, request: CreateUserRequest, context: grpc.ServicerContext) -> UserResponse: + # VULNERABLE: No validation beyond protobuf types + email = request.email # Could be malformed + password = request.password # Could be weak + return create_user_in_db(email, password) +``` + +**Why**: Protobuf only provides type validation, not business logic validation. Unvalidated input enables injection attacks and business logic bypass. + +**Refs**: OWASP A03:2025, CWE-20 + +--- + +### Rule: Validate Resource Identifiers + +**Level**: `strict` + +**When**: Using field values for resource access. + +**Do**: +```python +import re +from uuid import UUID + +class DocumentServicer(noteflow_pb2_grpc.DocumentServiceServicer): + def GetDocument(self, request: GetDocumentRequest, context: grpc.ServicerContext) -> DocumentResponse: + # Validate UUID format + try: + doc_id = UUID(request.document_id) + except ValueError: + context.set_code(grpc.StatusCode.INVALID_ARGUMENT) + context.set_details("Invalid document ID format") + return DocumentResponse() + + # Validate filename patterns if applicable + if hasattr(request, 'filename') and request.filename: + if not re.match(r'^[a-zA-Z0-9_-]+\.[a-z]+$', request.filename): + context.set_code(grpc.StatusCode.INVALID_ARGUMENT) + context.set_details("Invalid filename format") + return DocumentResponse() + + return get_document_by_id(str(doc_id)) +``` + +**Don't**: +```python +def GetDocument(self, request: GetDocumentRequest, context: grpc.ServicerContext) -> DocumentResponse: + # VULNERABLE: No ID validation + document = db.query(Document).filter(Document.id == request.document_id).first() + return document +``` + +**Why**: Unvalidated identifiers enable path traversal, injection, and enumeration attacks. + +**Refs**: CWE-22, OWASP A01:2025 + +--- + +## Authentication + +### Rule: Implement Secure gRPC Authentication + +**Level**: `strict` + +**When**: Securing gRPC endpoints. + +**Do**: +```python +import grpc +from grpc import aio +from jose import JWTError, jwt +from datetime import datetime, timedelta + +class AuthInterceptor(grpc.aio.ServerInterceptor): + def __init__(self, secret_key: str): + self.secret_key = secret_key + + async def intercept_service(self, continuation, handler_call_details): + # Extract token from metadata + metadata = dict(handler_call_details.invocation_metadata) + token = metadata.get('authorization', '').replace('Bearer ', '') + + if not token: + await abort(grpc.StatusCode.UNAUTHENTICATED, 'Missing token') + + try: + payload = jwt.decode(token, self.secret_key, algorithms=['HS256']) + user_id = payload.get('sub') + if user_id is None: + await abort(grpc.StatusCode.UNAUTHENTICATED, 'Invalid token') + except JWTError: + await abort(grpc.StatusCode.UNAUTHENTICATED, 'Invalid token') + + # Add user context to handler + handler_call_details = handler_call_details._replace( + credentials=grpc.aio.ServerCredentials(user_id=user_id) + ) + + return await continuation(handler_call_details) + +# Server setup +async def create_server(): + server = aio.server( + interceptors=[AuthInterceptor(os.environ['JWT_SECRET'])] + ) + noteflow_pb2_grpc.add_UserServiceServicer_to_server(UserServicer(), server) + return server +``` + +**Don't**: +```python +# VULNERABLE: No authentication +server = aio.server() +noteflow_pb2_grpc.add_UserServiceServicer_to_server(UserServicer(), server) + +# VULNERABLE: Hardcoded secret +SECRET_KEY = "mysecretkey123" +``` + +**Why**: gRPC without proper authentication allows unauthorized access to all endpoints. + +**Refs**: CWE-287, OWASP A07:2025 + +--- + +### Rule: Implement Rate Limiting Interceptor + +**Level**: `warning` + +**When**: Exposing authentication or resource-intensive endpoints. + +**Do**: +```python +from collections import defaultdict, deque +from time import time +import grpc + +class RateLimitInterceptor(grpc.aio.ServerInterceptor): + def __init__(self, requests_per_minute: int = 60): + self.requests_per_minute = requests_per_minute + self.requests = defaultdict(deque) + + async def intercept_service(self, continuation, handler_call_details): + # Extract client identifier from metadata or peer + client_id = handler_call_details.peer.split(':')[-1] # Simplified + + now = time() + client_requests = self.requests[client_id] + + # Clean old requests + while client_requests and client_requests[0] < now - 60: + client_requests.popleft() + + if len(client_requests) >= self.requests_per_minute: + await abort(grpc.StatusCode.RESOURCE_EXHAUSTED, 'Rate limit exceeded') + + client_requests.append(now) + return await continuation(handler_call_details) + +async def abort(status: grpc.StatusCode, message: str): + """Helper to abort gRPC calls with proper error handling.""" + raise grpc.aio.AioRpcError(status, message, None, None, None) +``` + +**Don't**: +```python +# VULNERABLE: No rate limiting +server = aio.server() +noteflow_pb2_grpc.add_UserServiceServicer_to_server(UserServicer(), server) +``` + +**Why**: Missing rate limits enable brute force attacks, credential stuffing, and DoS. + +**Refs**: CWE-307, OWASP A07:2025 + +--- + +## Authorization + +### Rule: Implement Resource-Level Authorization + +**Level**: `strict` + +**When**: Accessing user-owned or permission-restricted resources. + +**Do**: +```python +class DocumentServicer(noteflow_pb2_grpc.DocumentServiceServicer): + def GetDocument(self, request: GetDocumentRequest, context: grpc.ServicerContext) -> DocumentResponse: + # Get user from authentication context + user_id = get_user_from_context(context) + + document = get_document_by_id(request.document_id) + if not document: + context.set_code(grpc.StatusCode.NOT_FOUND) + context.set_details('Document not found') + return DocumentResponse() + + # Check ownership or permission + if document.owner_id != user_id and not is_admin(user_id): + context.set_code(grpc.StatusCode.PERMISSION_DENIED) + context.set_details('Access denied') + return DocumentResponse() + + return document_to_proto(document) + +def get_user_from_context(context: grpc.ServicerContext) -> str: + """Extract authenticated user ID from context.""" + # Implementation depends on auth interceptor setup + return context.user_id if hasattr(context, 'user_id') else None +``` + +**Don't**: +```python +def GetDocument(self, request: GetDocumentRequest, context: grpc.ServicerContext) -> DocumentResponse: + # VULNERABLE: No authorization check (IDOR) + document = get_document_by_id(request.document_id) + return document_to_proto(document) +``` + +**Why**: Missing authorization checks enable IDOR attacks where users access others' data. + +**Refs**: CWE-862, OWASP A01:2025 + +--- + +## Database Security + +### Rule: Use ORM with Parameterized Queries + +**Level**: `strict` + +**When**: Querying databases. + +**Do**: +```python +from sqlalchemy.orm import Session +from sqlalchemy import text + +class UserRepository: + def __init__(self, session: Session): + self.session = session + + def get_user_by_email(self, email: str): + # ORM queries are safe + return self.session.query(User).filter(User.email == email).first() + + def get_user_with_raw_sql(self, user_id: str): + # Raw SQL with parameters + result = self.session.execute( + text("SELECT * FROM users WHERE id = :user_id"), + {"user_id": user_id} + ) + return result.fetchone() + + def create_user(self, user_data: dict): + # Safe parameterized insertion + user = User(**user_data) + self.session.add(user) + self.session.commit() + return user +``` + +**Don't**: +```python +# VULNERABLE: SQL injection +query = f"SELECT * FROM users WHERE email = '{email}'" +result = db.execute(query) + +# VULNERABLE: String formatting +db.execute(f"DELETE FROM users WHERE id = {user_id}") +``` + +**Why**: SQL injection allows attackers to read, modify, or delete database data. + +**Refs**: CWE-89, OWASP A03:2025 + +--- + +## Response Security + +### Rule: Don't Expose Sensitive Data in Responses + +**Level**: `strict` + +**When**: Returning user or system data. + +**Do**: +```python +from noteflow_pb2 import UserResponse, DocumentResponse + +def user_to_proto(user: User) -> UserResponse: + """Convert domain User to protobuf, excluding sensitive fields.""" + return UserResponse( + id=user.id, + email=user.email, + name=user.name, + # password_hash intentionally excluded + created_at=user.created_at.isoformat() + ) + +def document_to_proto(document: Document) -> DocumentResponse: + """Convert domain Document to protobuf with field filtering.""" + return DocumentResponse( + id=document.id, + title=document.title, + content=document.content, + owner_id=document.owner_id, + # internal_metadata intentionally excluded + ) + +class UserServicer(noteflow_pb2_grpc.UserServiceServicer): + def GetUser(self, request: GetUserRequest, context: grpc.ServicerContext) -> UserResponse: + user = get_user_by_id(request.user_id) + if not user: + context.set_code(grpc.StatusCode.NOT_FOUND) + return UserResponse() + + return user_to_proto(user) # Sensitive data filtered +``` + +**Don't**: +```python +def GetUser(self, request: GetUserRequest, context: grpc.ServicerContext) -> UserResponse: + # VULNERABLE: Exposes all fields including sensitive ones + user = get_user_by_id(request.user_id) + return UserResponse( + id=user.id, + email=user.email, + password_hash=user.password_hash, # Sensitive! + internal_id=user.internal_id # Internal! + ) +``` + +**Why**: Exposing internal fields leaks sensitive data like password hashes or internal IDs. + +**Refs**: CWE-200, OWASP A01:2025 + +--- + +## Error Handling + +### Rule: Use Secure Error Handling + +**Level**: `warning` + +**When**: Handling errors in production. + +**Do**: +```python +import grpc +import logging + +logger = logging.getLogger(__name__) + +class SafeUserServicer(noteflow_pb2_grpc.UserServiceServicer): + def CreateUser(self, request: CreateUserRequest, context: grpc.ServicerContext) -> UserResponse: + try: + # Business logic here + user = create_user_in_db(request) + return user_to_proto(user) + except ValueError as e: + # Client error - safe to expose + context.set_code(grpc.StatusCode.INVALID_ARGUMENT) + context.set_details(str(e)) + return UserResponse() + except PermissionError as e: + context.set_code(grpc.StatusCode.PERMISSION_DENIED) + context.set_details("Access denied") + return UserResponse() + except Exception as e: + # Server error - log details, return generic message + logger.exception(f"Unexpected error in CreateUser: {e}") + context.set_code(grpc.StatusCode.INTERNAL) + context.set_details("Internal server error") + return UserResponse() + +def error_handler(context, error: Exception): + """Centralized error handling for gRPC services.""" + logger.exception(f"Unhandled exception: {error}") + context.set_code(grpc.StatusCode.INTERNAL) + context.set_details("Internal server error") +``` + +**Don't**: +```python +# VULNERABLE: Exposes stack traces +def CreateUser(self, request: CreateUserRequest, context: grpc.ServicerContext) -> UserResponse: + try: + return create_user_in_db(request) + except Exception as e: + # Exposes internal error details + context.set_code(grpc.StatusCode.INTERNAL) + context.set_details(f"Error: {str(e)}\n{traceback.format_exc()}") + return UserResponse() +``` + +**Why**: Stack traces expose internal paths, library versions, and code structure. + +**Refs**: CWE-209, OWASP A05:2025 + +--- + +## Streaming Security + +### Rule: Secure Streaming Connections + +**Level**: `strict` + +**When**: Implementing bidirectional or server streaming. + +**Do**: +```python +import grpc +from asyncio import Queue + +class StreamingServicer(noteflow_pb2_grpc.StreamingServiceServicer): + async def StreamAudio(self, request_iterator, context: grpc.ServicerContext): + # Authenticate before streaming + user_id = get_user_from_context(context) + if not user_id: + await abort(grpc.StatusCode.UNAUTHENTICATED, 'Authentication required') + + # Validate stream parameters + try: + metadata = dict(context.invocation_metadata) + max_duration = int(metadata.get('max-duration', '300')) # 5 min default + if max_duration > 3600: # 1 hour max + await abort(grpc.StatusCode.INVALID_ARGUMENT, 'Duration too long') + except (ValueError, TypeError): + await abort(grpc.StatusCode.INVALID_ARGUMENT, 'Invalid duration') + + # Rate limit streaming data + bytes_received = 0 + max_bytes = 100 * 1024 * 1024 # 100MB limit + + try: + async for request in request_iterator: + bytes_received += len(request.audio_data) + if bytes_received > max_bytes: + await abort(grpc.StatusCode.RESOURCE_EXHAUSTED, 'Stream too large') + + # Process audio chunk + processed = await process_audio_chunk(request.audio_data) + yield AudioChunkResponse(data=processed) + + except grpc.RpcError as e: + logger.warning(f"Stream error: {e}") + raise + except Exception as e: + logger.exception(f"Unexpected stream error: {e}") + await abort(grpc.StatusCode.INTERNAL, 'Stream processing failed') +``` + +**Don't**: +```python +# VULNERABLE: No authentication or limits +async def StreamAudio(self, request_iterator, context): + async for request in request_iterator: + # Unlimited data processing + result = process_audio(request.audio_data) + yield AudioChunkResponse(data=result) +``` + +**Why**: Unsecured streams can be exploited for DoS attacks, data exfiltration, and resource exhaustion. + +**Refs**: CWE-400, CWE-770, OWASP A05:2025 + +--- + +## AI/ML gRPC Security + +### Rule: Secure ML Model Inference Endpoints + +**Level**: `strict` + +**When**: Building gRPC services for ML model inference. + +**Do**: +```python +from noteflow_pb2 import InferenceRequest, InferenceResponse +import torch + +class MLServicer(noteflow_pb2_grpc.MLServiceServicer): + def __init__(self): + self.model = self.load_model_safely() + + def load_model_safely(self): + """Load model with security validation.""" + model_path = os.environ.get("MODEL_PATH") + if not model_path: + raise ValueError("MODEL_PATH not configured") + + # Use safe serialization + model = torch.jit.load(model_path) + model.eval() + return model + + async def Predict(self, request: InferenceRequest, context: grpc.ServicerContext) -> InferenceResponse: + # Input validation + if len(request.input_data) == 0: + context.set_code(grpc.StatusCode.INVALID_ARGUMENT) + context.set_details("Empty input data") + return InferenceResponse() + + if len(request.input_data) > 10000: # Size limit + context.set_code(grpc.StatusCode.INVALID_ARGUMENT) + context.set_details("Input too large") + return InferenceResponse() + + # Rate limiting check + user_id = get_user_from_context(context) + if not await self.check_inference_limit(user_id): + context.set_code(grpc.StatusCode.RESOURCE_EXHAUSTED) + context.set_details("Inference rate limit exceeded") + return InferenceResponse() + + try: + # Safe inference + with torch.no_grad(): + input_tensor = torch.tensor(request.input_data) + output = self.model(input_tensor) + + # Output validation/filtering + filtered_output = self.validate_output(output) + + return InferenceResponse( + prediction=filtered_output.tolist(), + confidence=self.calculate_confidence(output) + ) + except Exception as e: + logger.exception(f"Inference error: {e}") + context.set_code(grpc.StatusCode.INTERNAL) + context.set_details("Inference failed") + return InferenceResponse() + + def validate_output(self, output): + """Filter sensitive data from model output.""" + # Check for and filter sensitive patterns + # Implementation depends on model type + return output +``` + +**Don't**: +```python +# VULNERABLE: No input validation or rate limiting +async def Predict(self, request: InferenceRequest, context): + input_tensor = torch.tensor(request.input_data) + output = self.model(input_tensor) + return InferenceResponse(prediction=output.tolist()) +``` + +**Why**: ML endpoints need strict validation and rate limiting to prevent abuse, resource exhaustion, and data leakage. + +**Refs**: OWASP LLM01, OWASP LLM04, CWE-400 + +--- + +## Quick Reference + +| Rule | Level | CWE | +|------|-------|-----| +| Protobuf validation | strict | CWE-20 | +| Resource identifier validation | strict | CWE-22 | +| gRPC authentication | strict | CWE-287 | +| Rate limiting interceptor | warning | CWE-307 | +| Resource authorization | strict | CWE-862 | +| ORM/parameterized queries | strict | CWE-89 | +| Response filtering | strict | CWE-200 | +| Secure error handling | warning | CWE-209 | +| Streaming security | strict | CWE-400, CWE-770 | +| ML inference security | strict | OWASP LLM01, CWE-400 | + +--- + +## Version History + +- **v1.0.0** - Initial gRPC security rules adapted from FastAPI \ No newline at end of file diff --git a/src/noteflow/grpc/CLAUDE.md b/src/noteflow/grpc/CLAUDE.md new file mode 100644 index 0000000..0e97023 --- /dev/null +++ b/src/noteflow/grpc/CLAUDE.md @@ -0,0 +1,638 @@ +# gRPC Security Rules + +Security rules for gRPC development in NoteFlow. + +## Prerequisites + +- `../../.agent/rules/_core/owasp-2025.md` - Core web security +- `../../.agent/rules/languages/python/CLAUDE.md` - Python security +- NoteFlow gRPC schema: `proto/noteflow.proto` - Shared contract + +--- + +## Input Validation + +### Rule: Use Protobuf Message Validation + +**Level**: `strict` + +**When**: Accepting request data from clients. + +**Do**: +```python +import grpc +from pydantic import BaseModel, Field, validator +from noteflow_pb2 import CreateUserRequest, UserResponse + +class PydanticUserCreate(BaseModel): + email: str = Field(..., regex=r'^[^@]+@[^@]+\.[^@]+$') + password: str = Field(..., min_length=8, max_length=128) + age: int = Field(..., ge=0, le=150) + + @validator('password') + def password_strength(cls, v): + if not any(c.isupper() for c in v): + raise ValueError('Password must contain uppercase') + if not any(c.isdigit() for c in v): + raise ValueError('Password must contain digit') + return v + +class UserServicer(noteflow_pb2_grpc.UserServiceServicer): + def CreateUser(self, request: CreateUserRequest, context: grpc.ServicerContext) -> UserResponse: + # Validate with Pydantic before processing + try: + validated_data = PydanticUserCreate( + email=request.email, + password=request.password, + age=request.age + ) + except ValueError as e: + context.set_code(grpc.StatusCode.INVALID_ARGUMENT) + context.set_details(str(e)) + return UserResponse() + + # Process validated data + return create_user_in_db(validated_data.dict()) +``` + +**Don't**: +```python +class UserServicer(noteflow_pb2_grpc.UserServiceServicer): + def CreateUser(self, request: CreateUserRequest, context: grpc.ServicerContext) -> UserResponse: + # VULNERABLE: No validation beyond protobuf types + email = request.email # Could be malformed + password = request.password # Could be weak + return create_user_in_db(email, password) +``` + +**Why**: Protobuf only provides type validation, not business logic validation. Unvalidated input enables injection attacks and business logic bypass. + +**Refs**: OWASP A03:2025, CWE-20 + +--- + +### Rule: Validate Resource Identifiers + +**Level**: `strict` + +**When**: Using field values for resource access. + +**Do**: +```python +import re +from uuid import UUID + +class DocumentServicer(noteflow_pb2_grpc.DocumentServiceServicer): + def GetDocument(self, request: GetDocumentRequest, context: grpc.ServicerContext) -> DocumentResponse: + # Validate UUID format + try: + doc_id = UUID(request.document_id) + except ValueError: + context.set_code(grpc.StatusCode.INVALID_ARGUMENT) + context.set_details("Invalid document ID format") + return DocumentResponse() + + # Validate filename patterns if applicable + if hasattr(request, 'filename') and request.filename: + if not re.match(r'^[a-zA-Z0-9_-]+\.[a-z]+$', request.filename): + context.set_code(grpc.StatusCode.INVALID_ARGUMENT) + context.set_details("Invalid filename format") + return DocumentResponse() + + return get_document_by_id(str(doc_id)) +``` + +**Don't**: +```python +def GetDocument(self, request: GetDocumentRequest, context: grpc.ServicerContext) -> DocumentResponse: + # VULNERABLE: No ID validation + document = db.query(Document).filter(Document.id == request.document_id).first() + return document +``` + +**Why**: Unvalidated identifiers enable path traversal, injection, and enumeration attacks. + +**Refs**: CWE-22, OWASP A01:2025 + +--- + +## Authentication + +### Rule: Implement Secure gRPC Authentication + +**Level**: `strict` + +**When**: Securing gRPC endpoints. + +**Do**: +```python +import grpc +from grpc import aio +from jose import JWTError, jwt +from datetime import datetime, timedelta + +class AuthInterceptor(grpc.aio.ServerInterceptor): + def __init__(self, secret_key: str): + self.secret_key = secret_key + + async def intercept_service(self, continuation, handler_call_details): + # Extract token from metadata + metadata = dict(handler_call_details.invocation_metadata) + token = metadata.get('authorization', '').replace('Bearer ', '') + + if not token: + await abort(grpc.StatusCode.UNAUTHENTICATED, 'Missing token') + + try: + payload = jwt.decode(token, self.secret_key, algorithms=['HS256']) + user_id = payload.get('sub') + if user_id is None: + await abort(grpc.StatusCode.UNAUTHENTICATED, 'Invalid token') + except JWTError: + await abort(grpc.StatusCode.UNAUTHENTICATED, 'Invalid token') + + # Add user context to handler + handler_call_details = handler_call_details._replace( + credentials=grpc.aio.ServerCredentials(user_id=user_id) + ) + + return await continuation(handler_call_details) + +# Server setup +async def create_server(): + server = aio.server( + interceptors=[AuthInterceptor(os.environ['JWT_SECRET'])] + ) + noteflow_pb2_grpc.add_UserServiceServicer_to_server(UserServicer(), server) + return server +``` + +**Don't**: +```python +# VULNERABLE: No authentication +server = aio.server() +noteflow_pb2_grpc.add_UserServiceServicer_to_server(UserServicer(), server) + +# VULNERABLE: Hardcoded secret +SECRET_KEY = "mysecretkey123" +``` + +**Why**: gRPC without proper authentication allows unauthorized access to all endpoints. + +**Refs**: CWE-287, OWASP A07:2025 + +--- + +### Rule: Implement Rate Limiting Interceptor + +**Level**: `warning` + +**When**: Exposing authentication or resource-intensive endpoints. + +**Do**: +```python +from collections import defaultdict, deque +from time import time +import grpc + +class RateLimitInterceptor(grpc.aio.ServerInterceptor): + def __init__(self, requests_per_minute: int = 60): + self.requests_per_minute = requests_per_minute + self.requests = defaultdict(deque) + + async def intercept_service(self, continuation, handler_call_details): + # Extract client identifier from metadata or peer + client_id = handler_call_details.peer.split(':')[-1] # Simplified + + now = time() + client_requests = self.requests[client_id] + + # Clean old requests + while client_requests and client_requests[0] < now - 60: + client_requests.popleft() + + if len(client_requests) >= self.requests_per_minute: + await abort(grpc.StatusCode.RESOURCE_EXHAUSTED, 'Rate limit exceeded') + + client_requests.append(now) + return await continuation(handler_call_details) + +async def abort(status: grpc.StatusCode, message: str): + """Helper to abort gRPC calls with proper error handling.""" + raise grpc.aio.AioRpcError(status, message, None, None, None) +``` + +**Don't**: +```python +# VULNERABLE: No rate limiting +server = aio.server() +noteflow_pb2_grpc.add_UserServiceServicer_to_server(UserServicer(), server) +``` + +**Why**: Missing rate limits enable brute force attacks, credential stuffing, and DoS. + +**Refs**: CWE-307, OWASP A07:2025 + +--- + +## Authorization + +### Rule: Implement Resource-Level Authorization + +**Level**: `strict` + +**When**: Accessing user-owned or permission-restricted resources. + +**Do**: +```python +class DocumentServicer(noteflow_pb2_grpc.DocumentServiceServicer): + def GetDocument(self, request: GetDocumentRequest, context: grpc.ServicerContext) -> DocumentResponse: + # Get user from authentication context + user_id = get_user_from_context(context) + + document = get_document_by_id(request.document_id) + if not document: + context.set_code(grpc.StatusCode.NOT_FOUND) + context.set_details('Document not found') + return DocumentResponse() + + # Check ownership or permission + if document.owner_id != user_id and not is_admin(user_id): + context.set_code(grpc.StatusCode.PERMISSION_DENIED) + context.set_details('Access denied') + return DocumentResponse() + + return document_to_proto(document) + +def get_user_from_context(context: grpc.ServicerContext) -> str: + """Extract authenticated user ID from context.""" + # Implementation depends on auth interceptor setup + return context.user_id if hasattr(context, 'user_id') else None +``` + +**Don't**: +```python +def GetDocument(self, request: GetDocumentRequest, context: grpc.ServicerContext) -> DocumentResponse: + # VULNERABLE: No authorization check (IDOR) + document = get_document_by_id(request.document_id) + return document_to_proto(document) +``` + +**Why**: Missing authorization checks enable IDOR attacks where users access others' data. + +**Refs**: CWE-862, OWASP A01:2025 + +--- + +## Database Security + +### Rule: Use ORM with Parameterized Queries + +**Level**: `strict` + +**When**: Querying databases. + +**Do**: +```python +from sqlalchemy.orm import Session +from sqlalchemy import text + +class UserRepository: + def __init__(self, session: Session): + self.session = session + + def get_user_by_email(self, email: str): + # ORM queries are safe + return self.session.query(User).filter(User.email == email).first() + + def get_user_with_raw_sql(self, user_id: str): + # Raw SQL with parameters + result = self.session.execute( + text("SELECT * FROM users WHERE id = :user_id"), + {"user_id": user_id} + ) + return result.fetchone() + + def create_user(self, user_data: dict): + # Safe parameterized insertion + user = User(**user_data) + self.session.add(user) + self.session.commit() + return user +``` + +**Don't**: +```python +# VULNERABLE: SQL injection +query = f"SELECT * FROM users WHERE email = '{email}'" +result = db.execute(query) + +# VULNERABLE: String formatting +db.execute(f"DELETE FROM users WHERE id = {user_id}") +``` + +**Why**: SQL injection allows attackers to read, modify, or delete database data. + +**Refs**: CWE-89, OWASP A03:2025 + +--- + +## Response Security + +### Rule: Don't Expose Sensitive Data in Responses + +**Level**: `strict` + +**When**: Returning user or system data. + +**Do**: +```python +from noteflow_pb2 import UserResponse, DocumentResponse + +def user_to_proto(user: User) -> UserResponse: + """Convert domain User to protobuf, excluding sensitive fields.""" + return UserResponse( + id=user.id, + email=user.email, + name=user.name, + # password_hash intentionally excluded + created_at=user.created_at.isoformat() + ) + +def document_to_proto(document: Document) -> DocumentResponse: + """Convert domain Document to protobuf with field filtering.""" + return DocumentResponse( + id=document.id, + title=document.title, + content=document.content, + owner_id=document.owner_id, + # internal_metadata intentionally excluded + ) + +class UserServicer(noteflow_pb2_grpc.UserServiceServicer): + def GetUser(self, request: GetUserRequest, context: grpc.ServicerContext) -> UserResponse: + user = get_user_by_id(request.user_id) + if not user: + context.set_code(grpc.StatusCode.NOT_FOUND) + return UserResponse() + + return user_to_proto(user) # Sensitive data filtered +``` + +**Don't**: +```python +def GetUser(self, request: GetUserRequest, context: grpc.ServicerContext) -> UserResponse: + # VULNERABLE: Exposes all fields including sensitive ones + user = get_user_by_id(request.user_id) + return UserResponse( + id=user.id, + email=user.email, + password_hash=user.password_hash, # Sensitive! + internal_id=user.internal_id # Internal! + ) +``` + +**Why**: Exposing internal fields leaks sensitive data like password hashes or internal IDs. + +**Refs**: CWE-200, OWASP A01:2025 + +--- + +## Error Handling + +### Rule: Use Secure Error Handling + +**Level**: `warning` + +**When**: Handling errors in production. + +**Do**: +```python +import grpc +import logging + +logger = logging.getLogger(__name__) + +class SafeUserServicer(noteflow_pb2_grpc.UserServiceServicer): + def CreateUser(self, request: CreateUserRequest, context: grpc.ServicerContext) -> UserResponse: + try: + # Business logic here + user = create_user_in_db(request) + return user_to_proto(user) + except ValueError as e: + # Client error - safe to expose + context.set_code(grpc.StatusCode.INVALID_ARGUMENT) + context.set_details(str(e)) + return UserResponse() + except PermissionError as e: + context.set_code(grpc.StatusCode.PERMISSION_DENIED) + context.set_details("Access denied") + return UserResponse() + except Exception as e: + # Server error - log details, return generic message + logger.exception(f"Unexpected error in CreateUser: {e}") + context.set_code(grpc.StatusCode.INTERNAL) + context.set_details("Internal server error") + return UserResponse() + +def error_handler(context, error: Exception): + """Centralized error handling for gRPC services.""" + logger.exception(f"Unhandled exception: {error}") + context.set_code(grpc.StatusCode.INTERNAL) + context.set_details("Internal server error") +``` + +**Don't**: +```python +# VULNERABLE: Exposes stack traces +def CreateUser(self, request: CreateUserRequest, context: grpc.ServicerContext) -> UserResponse: + try: + return create_user_in_db(request) + except Exception as e: + # Exposes internal error details + context.set_code(grpc.StatusCode.INTERNAL) + context.set_details(f"Error: {str(e)}\n{traceback.format_exc()}") + return UserResponse() +``` + +**Why**: Stack traces expose internal paths, library versions, and code structure. + +**Refs**: CWE-209, OWASP A05:2025 + +--- + +## Streaming Security + +### Rule: Secure Streaming Connections + +**Level**: `strict` + +**When**: Implementing bidirectional or server streaming. + +**Do**: +```python +import grpc +from asyncio import Queue + +class StreamingServicer(noteflow_pb2_grpc.StreamingServiceServicer): + async def StreamAudio(self, request_iterator, context: grpc.ServicerContext): + # Authenticate before streaming + user_id = get_user_from_context(context) + if not user_id: + await abort(grpc.StatusCode.UNAUTHENTICATED, 'Authentication required') + + # Validate stream parameters + try: + metadata = dict(context.invocation_metadata) + max_duration = int(metadata.get('max-duration', '300')) # 5 min default + if max_duration > 3600: # 1 hour max + await abort(grpc.StatusCode.INVALID_ARGUMENT, 'Duration too long') + except (ValueError, TypeError): + await abort(grpc.StatusCode.INVALID_ARGUMENT, 'Invalid duration') + + # Rate limit streaming data + bytes_received = 0 + max_bytes = 100 * 1024 * 1024 # 100MB limit + + try: + async for request in request_iterator: + bytes_received += len(request.audio_data) + if bytes_received > max_bytes: + await abort(grpc.StatusCode.RESOURCE_EXHAUSTED, 'Stream too large') + + # Process audio chunk + processed = await process_audio_chunk(request.audio_data) + yield AudioChunkResponse(data=processed) + + except grpc.RpcError as e: + logger.warning(f"Stream error: {e}") + raise + except Exception as e: + logger.exception(f"Unexpected stream error: {e}") + await abort(grpc.StatusCode.INTERNAL, 'Stream processing failed') +``` + +**Don't**: +```python +# VULNERABLE: No authentication or limits +async def StreamAudio(self, request_iterator, context): + async for request in request_iterator: + # Unlimited data processing + result = process_audio(request.audio_data) + yield AudioChunkResponse(data=result) +``` + +**Why**: Unsecured streams can be exploited for DoS attacks, data exfiltration, and resource exhaustion. + +**Refs**: CWE-400, CWE-770, OWASP A05:2025 + +--- + +## AI/ML gRPC Security + +### Rule: Secure ML Model Inference Endpoints + +**Level**: `strict` + +**When**: Building gRPC services for ML model inference. + +**Do**: +```python +from noteflow_pb2 import InferenceRequest, InferenceResponse +import torch + +class MLServicer(noteflow_pb2_grpc.MLServiceServicer): + def __init__(self): + self.model = self.load_model_safely() + + def load_model_safely(self): + """Load model with security validation.""" + model_path = os.environ.get("MODEL_PATH") + if not model_path: + raise ValueError("MODEL_PATH not configured") + + # Use safe serialization + model = torch.jit.load(model_path) + model.eval() + return model + + async def Predict(self, request: InferenceRequest, context: grpc.ServicerContext) -> InferenceResponse: + # Input validation + if len(request.input_data) == 0: + context.set_code(grpc.StatusCode.INVALID_ARGUMENT) + context.set_details("Empty input data") + return InferenceResponse() + + if len(request.input_data) > 10000: # Size limit + context.set_code(grpc.StatusCode.INVALID_ARGUMENT) + context.set_details("Input too large") + return InferenceResponse() + + # Rate limiting check + user_id = get_user_from_context(context) + if not await self.check_inference_limit(user_id): + context.set_code(grpc.StatusCode.RESOURCE_EXHAUSTED) + context.set_details("Inference rate limit exceeded") + return InferenceResponse() + + try: + # Safe inference + with torch.no_grad(): + input_tensor = torch.tensor(request.input_data) + output = self.model(input_tensor) + + # Output validation/filtering + filtered_output = self.validate_output(output) + + return InferenceResponse( + prediction=filtered_output.tolist(), + confidence=self.calculate_confidence(output) + ) + except Exception as e: + logger.exception(f"Inference error: {e}") + context.set_code(grpc.StatusCode.INTERNAL) + context.set_details("Inference failed") + return InferenceResponse() + + def validate_output(self, output): + """Filter sensitive data from model output.""" + # Check for and filter sensitive patterns + # Implementation depends on model type + return output +``` + +**Don't**: +```python +# VULNERABLE: No input validation or rate limiting +async def Predict(self, request: InferenceRequest, context): + input_tensor = torch.tensor(request.input_data) + output = self.model(input_tensor) + return InferenceResponse(prediction=output.tolist()) +``` + +**Why**: ML endpoints need strict validation and rate limiting to prevent abuse, resource exhaustion, and data leakage. + +**Refs**: OWASP LLM01, OWASP LLM04, CWE-400 + +--- + +## Quick Reference + +| Rule | Level | CWE | +|------|-------|-----| +| Protobuf validation | strict | CWE-20 | +| Resource identifier validation | strict | CWE-22 | +| gRPC authentication | strict | CWE-287 | +| Rate limiting interceptor | warning | CWE-307 | +| Resource authorization | strict | CWE-862 | +| ORM/parameterized queries | strict | CWE-89 | +| Response filtering | strict | CWE-200 | +| Secure error handling | warning | CWE-209 | +| Streaming security | strict | CWE-400, CWE-770 | +| ML inference security | strict | OWASP LLM01, CWE-400 | + +--- + +## Version History + +- **v1.0.0** - Initial gRPC security rules adapted from FastAPI \ No newline at end of file diff --git a/src/noteflow/grpc/_client_mixins/__init__.py b/src/noteflow/grpc/_client_mixins/__init__.py deleted file mode 100644 index 30f720b..0000000 --- a/src/noteflow/grpc/_client_mixins/__init__.py +++ /dev/null @@ -1,20 +0,0 @@ -"""Client mixins for modular NoteFlowClient composition.""" - -from noteflow.grpc._client_mixins.annotation import AnnotationClientMixin -from noteflow.grpc._client_mixins.converters import proto_to_annotation_info, proto_to_meeting_info -from noteflow.grpc._client_mixins.diarization import DiarizationClientMixin -from noteflow.grpc._client_mixins.export import ExportClientMixin -from noteflow.grpc._client_mixins.meeting import MeetingClientMixin -from noteflow.grpc._client_mixins.protocols import ClientHost -from noteflow.grpc._client_mixins.streaming import StreamingClientMixin - -__all__ = [ - "AnnotationClientMixin", - "ClientHost", - "DiarizationClientMixin", - "ExportClientMixin", - "MeetingClientMixin", - "StreamingClientMixin", - "proto_to_annotation_info", - "proto_to_meeting_info", -] diff --git a/src/noteflow/grpc/_types.py b/src/noteflow/grpc/_types.py deleted file mode 100644 index 1153414..0000000 --- a/src/noteflow/grpc/_types.py +++ /dev/null @@ -1,108 +0,0 @@ -"""Data types for NoteFlow gRPC client operations.""" - -from __future__ import annotations - -from collections.abc import Callable -from dataclasses import dataclass - - -@dataclass -class TranscriptSegment: - """Transcript segment from server.""" - - segment_id: int - text: str - start_time: float - end_time: float - language: str - is_final: bool - speaker_id: str = "" - speaker_confidence: float = 0.0 - - -@dataclass -class ServerInfo: - """Server information.""" - - version: str - asr_model: str - asr_ready: bool - uptime_seconds: float - active_meetings: int - diarization_enabled: bool = False - diarization_ready: bool = False - system_ram_total_bytes: int | None = None - system_ram_available_bytes: int | None = None - gpu_vram_total_bytes: int | None = None - gpu_vram_available_bytes: int | None = None - - -@dataclass -class MeetingInfo: - """Meeting information.""" - - id: str - title: str - state: str - created_at: float - started_at: float - ended_at: float - duration_seconds: float - segment_count: int - - -@dataclass -class AnnotationInfo: - """Annotation information.""" - - id: str - meeting_id: str - annotation_type: str - text: str - start_time: float - end_time: float - segment_ids: list[int] - created_at: float - - -@dataclass -class ExportResult: - """Export result.""" - - content: str - format_name: str - file_extension: str - - -@dataclass -class DiarizationResult: - """Result of speaker diarization refinement.""" - - job_id: str - status: str - segments_updated: int - speaker_ids: list[str] - error_message: str = "" - - @property - def success(self) -> bool: - """Check if diarization succeeded.""" - return self.status == "completed" and not self.error_message - - @property - def is_terminal(self) -> bool: - """Check if job reached a terminal state.""" - return self.status in {"completed", "failed"} - - -@dataclass -class RenameSpeakerResult: - """Result of speaker rename operation.""" - - segments_updated: int - success: bool - - -# Callback types -TranscriptCallback = Callable[[TranscriptSegment], None] -ConnectionCallback = Callable[[bool, str], None] diff --git a/src/noteflow/grpc/client.py b/src/noteflow/grpc/client.py index 12be66e..036b136 100644 --- a/src/noteflow/grpc/client.py +++ b/src/noteflow/grpc/client.py @@ -8,16 +8,16 @@ from typing import TYPE_CHECKING, Final import grpc -from noteflow.grpc import _types -from noteflow.grpc._client_mixins import ( +from noteflow.grpc import types as grpc_types +from noteflow.grpc.client_mixins import ( AnnotationClientMixin, DiarizationClientMixin, ExportClientMixin, MeetingClientMixin, StreamingClientMixin, ) -from noteflow.grpc._config import STREAMING_CONFIG -from noteflow.grpc._types import ( +from noteflow.grpc.config.config import STREAMING_CONFIG +from noteflow.grpc.types import ( AnnotationInfo, DiarizationResult, ExportResult, @@ -33,7 +33,7 @@ if TYPE_CHECKING: import numpy as np from numpy.typing import NDArray - from noteflow.grpc._client_mixins.protocols import ClientHost, NoteFlowServiceStubProtocol + from noteflow.grpc.client_mixins.protocols import ClientHost, NoteFlowServiceStubProtocol logger = get_logger(__name__) @@ -68,8 +68,8 @@ class NoteFlowClient( def __init__( self, server_address: str = DEFAULT_SERVER, - on_transcript: _types.TranscriptCallback | None = None, - on_connection_change: _types.ConnectionCallback | None = None, + on_transcript: grpc_types.TranscriptCallback | None = None, + on_connection_change: grpc_types.ConnectionCallback | None = None, ) -> None: """Initialize the client. @@ -178,7 +178,7 @@ class NoteFlowClient( logger.info("Disconnected from server") self.notify_connection(False, "Disconnected") - def get_server_info(self) -> _types.ServerInfo | None: + def get_server_info(self) -> grpc_types.ServerInfo | None: """Get server information. Returns: @@ -189,7 +189,7 @@ class NoteFlowClient( try: response = self._stub.GetServerInfo(noteflow_pb2.ServerInfoRequest()) - return _types.ServerInfo( + return grpc_types.ServerInfo( version=response.version, asr_model=response.asr_model, asr_ready=response.asr_ready, diff --git a/src/noteflow/grpc/client_mixins/__init__.py b/src/noteflow/grpc/client_mixins/__init__.py new file mode 100644 index 0000000..944d296 --- /dev/null +++ b/src/noteflow/grpc/client_mixins/__init__.py @@ -0,0 +1,20 @@ +"""Client mixins for modular NoteFlowClient composition.""" + +from noteflow.grpc.client_mixins.annotation import AnnotationClientMixin +from noteflow.grpc.client_mixins.converters import proto_to_annotation_info, proto_to_meeting_info +from noteflow.grpc.client_mixins.diarization import DiarizationClientMixin +from noteflow.grpc.client_mixins.export import ExportClientMixin +from noteflow.grpc.client_mixins.meeting import MeetingClientMixin +from noteflow.grpc.client_mixins.protocols import ClientHost +from noteflow.grpc.client_mixins.streaming import StreamingClientMixin + +__all__ = [ + "AnnotationClientMixin", + "ClientHost", + "DiarizationClientMixin", + "ExportClientMixin", + "MeetingClientMixin", + "StreamingClientMixin", + "proto_to_annotation_info", + "proto_to_meeting_info", +] diff --git a/src/noteflow/grpc/client_mixins/_constants.py b/src/noteflow/grpc/client_mixins/_constants.py new file mode 100644 index 0000000..1416b1f --- /dev/null +++ b/src/noteflow/grpc/client_mixins/_constants.py @@ -0,0 +1,28 @@ +"""Constants for gRPC client mixin operations.""" + +from typing import Final + +# Meeting operations +OP_MEETING_CREATE: Final = "meeting_create" +OP_MEETING_CREATE_LEGACY: Final = "create_meeting" +OP_STOP_MEETING: Final = "stop_meeting" +OP_MEETING_FETCH: Final = "meeting_fetch" +OP_MEETING_FETCH_LEGACY: Final = "get_meeting" +OP_GET_MEETING_SEGMENTS: Final = "get_meeting_segments" +OP_LIST_MEETINGS: Final = "list_meetings" + +# Annotation operations +OP_ADD_ANNOTATION: Final = "add_annotation" +OP_GET_ANNOTATION: Final = "get_annotation" +OP_ANNOTATION_FETCH: Final = "annotation_fetch" +OP_LIST_ANNOTATIONS: Final = "list_annotations" +OP_UPDATE_ANNOTATION: Final = "update_annotation" +OP_DELETE_ANNOTATION: Final = "delete_annotation" + +# Diarization operations +OP_REFINE_SPEAKER_DIARIZATION: Final = "refine_speaker_diarization" +OP_GET_DIARIZATION_JOB_STATUS: Final = "get_diarization_job_status" +OP_RENAME_SPEAKER: Final = "rename_speaker" + +# Export operations +OP_EXPORT_TRANSCRIPT: Final = "export_transcript" diff --git a/src/noteflow/grpc/client_mixins/_error_handling.py b/src/noteflow/grpc/client_mixins/_error_handling.py new file mode 100644 index 0000000..3a91661 --- /dev/null +++ b/src/noteflow/grpc/client_mixins/_error_handling.py @@ -0,0 +1,77 @@ +"""Error handling utilities for gRPC client operations.""" + +from __future__ import annotations + +from typing import Never + +import grpc + +from noteflow.grpc.types import ClientErrorCode, ClientResult, Err +from noteflow.infrastructure.logging import get_logger + +logger = get_logger(__name__) + +_GRPC_TO_CLIENT_ERROR: dict[grpc.StatusCode, ClientErrorCode] = { + grpc.StatusCode.NOT_FOUND: ClientErrorCode.NOT_FOUND, + grpc.StatusCode.INVALID_ARGUMENT: ClientErrorCode.INVALID_ARGUMENT, + grpc.StatusCode.DEADLINE_EXCEEDED: ClientErrorCode.DEADLINE_EXCEEDED, + grpc.StatusCode.ALREADY_EXISTS: ClientErrorCode.ALREADY_EXISTS, + grpc.StatusCode.PERMISSION_DENIED: ClientErrorCode.PERMISSION_DENIED, + grpc.StatusCode.FAILED_PRECONDITION: ClientErrorCode.FAILED_PRECONDITION, + grpc.StatusCode.UNIMPLEMENTED: ClientErrorCode.UNIMPLEMENTED, + grpc.StatusCode.INTERNAL: ClientErrorCode.INTERNAL, + grpc.StatusCode.UNAVAILABLE: ClientErrorCode.UNAVAILABLE, + grpc.StatusCode.RESOURCE_EXHAUSTED: ClientErrorCode.UNAVAILABLE, +} + + +def grpc_error_to_result(exc: grpc.RpcError, operation: str) -> ClientResult[Never]: + """Convert gRPC error to ClientResult. + + Args: + exc: gRPC error exception. + operation: Name of the operation that failed (for logging). + + Returns: + ClientResult with error information. + """ + grpc_status: grpc.StatusCode | None = None + message = str(exc) + + if hasattr(exc, "code") and callable(exc.code): + grpc_status = exc.code() + if hasattr(exc, "details") and callable(exc.details) and (details := exc.details()): + if details: + message = details + + client_code = ( + _GRPC_TO_CLIENT_ERROR.get(grpc_status, ClientErrorCode.UNKNOWN) + if grpc_status + else ClientErrorCode.UNKNOWN + ) + + logger.error( + "gRPC operation failed", + operation=operation, + grpc_status=grpc_status.name if grpc_status else "UNKNOWN", + message=message, + ) + + return Err(code=client_code, message=message, grpc_status=grpc_status) + + +def not_connected_error(operation: str) -> ClientResult[Never]: + """Create error result for missing gRPC connection. + + Args: + operation: Name of the operation that failed (for logging). + + Returns: + ClientResult with NOT_CONNECTED error. + """ + logger.warning("gRPC operation attempted without connection", operation=operation) + return Err( + code=ClientErrorCode.NOT_CONNECTED, + message="Not connected to server", + grpc_status=None, + ) diff --git a/src/noteflow/grpc/_client_mixins/annotation.py b/src/noteflow/grpc/client_mixins/annotation.py similarity index 97% rename from src/noteflow/grpc/_client_mixins/annotation.py rename to src/noteflow/grpc/client_mixins/annotation.py index b3a330f..831a0a2 100644 --- a/src/noteflow/grpc/_client_mixins/annotation.py +++ b/src/noteflow/grpc/client_mixins/annotation.py @@ -7,16 +7,16 @@ from typing import TYPE_CHECKING, NotRequired, Required, TypedDict, Unpack, cast import grpc from noteflow.domain.constants.fields import ANNOTATION_TYPE, END_TIME, SEGMENT_IDS, START_TIME -from noteflow.grpc._client_mixins.converters import ( +from noteflow.grpc.client_mixins.converters import ( annotation_type_to_proto, proto_to_annotation_info, ) -from noteflow.grpc._types import AnnotationInfo +from noteflow.grpc.types import AnnotationInfo from noteflow.grpc.proto import noteflow_pb2 from noteflow.infrastructure.logging import get_client_rate_limiter, get_logger if TYPE_CHECKING: - from noteflow.grpc._client_mixins.protocols import ClientHost + from noteflow.grpc.client_mixins.protocols import ClientHost class _AnnotationCreateKwargs(TypedDict): diff --git a/src/noteflow/grpc/_client_mixins/converters.py b/src/noteflow/grpc/client_mixins/converters.py similarity index 98% rename from src/noteflow/grpc/_client_mixins/converters.py rename to src/noteflow/grpc/client_mixins/converters.py index 4ad9288..e0bea82 100644 --- a/src/noteflow/grpc/_client_mixins/converters.py +++ b/src/noteflow/grpc/client_mixins/converters.py @@ -17,7 +17,7 @@ from noteflow.domain.constants.fields import ( RISK, UNKNOWN, ) -from noteflow.grpc._types import AnnotationInfo, MeetingInfo +from noteflow.grpc.types import AnnotationInfo, MeetingInfo from noteflow.grpc.proto import noteflow_pb2 diff --git a/src/noteflow/grpc/_client_mixins/diarization.py b/src/noteflow/grpc/client_mixins/diarization.py similarity index 95% rename from src/noteflow/grpc/_client_mixins/diarization.py rename to src/noteflow/grpc/client_mixins/diarization.py index 6566f70..775fc23 100644 --- a/src/noteflow/grpc/_client_mixins/diarization.py +++ b/src/noteflow/grpc/client_mixins/diarization.py @@ -6,13 +6,13 @@ from typing import TYPE_CHECKING import grpc -from noteflow.grpc._client_mixins.converters import job_status_to_str -from noteflow.grpc._types import DiarizationResult, RenameSpeakerResult +from noteflow.grpc.client_mixins.converters import job_status_to_str +from noteflow.grpc.types import DiarizationResult, RenameSpeakerResult from noteflow.grpc.proto import noteflow_pb2 from noteflow.infrastructure.logging import get_client_rate_limiter, get_logger if TYPE_CHECKING: - from noteflow.grpc._client_mixins.protocols import ClientHost + from noteflow.grpc.client_mixins.protocols import ClientHost logger = get_logger(__name__) _rate_limiter = get_client_rate_limiter() diff --git a/src/noteflow/grpc/_client_mixins/export.py b/src/noteflow/grpc/client_mixins/export.py similarity index 89% rename from src/noteflow/grpc/_client_mixins/export.py rename to src/noteflow/grpc/client_mixins/export.py index 336d1bd..21a87d2 100644 --- a/src/noteflow/grpc/_client_mixins/export.py +++ b/src/noteflow/grpc/client_mixins/export.py @@ -6,13 +6,13 @@ from typing import TYPE_CHECKING import grpc -from noteflow.grpc._client_mixins.converters import export_format_to_proto -from noteflow.grpc._types import ExportResult +from noteflow.grpc.client_mixins.converters import export_format_to_proto +from noteflow.grpc.types import ExportResult from noteflow.grpc.proto import noteflow_pb2 from noteflow.infrastructure.logging import get_client_rate_limiter, get_logger if TYPE_CHECKING: - from noteflow.grpc._client_mixins.protocols import ClientHost + from noteflow.grpc.client_mixins.protocols import ClientHost logger = get_logger(__name__) _rate_limiter = get_client_rate_limiter() diff --git a/src/noteflow/grpc/_client_mixins/meeting.py b/src/noteflow/grpc/client_mixins/meeting.py similarity index 95% rename from src/noteflow/grpc/_client_mixins/meeting.py rename to src/noteflow/grpc/client_mixins/meeting.py index c4efb9b..a23a84b 100644 --- a/src/noteflow/grpc/_client_mixins/meeting.py +++ b/src/noteflow/grpc/client_mixins/meeting.py @@ -6,13 +6,13 @@ from typing import TYPE_CHECKING import grpc -from noteflow.grpc._client_mixins.converters import proto_to_meeting_info -from noteflow.grpc._types import MeetingInfo, TranscriptSegment +from noteflow.grpc.client_mixins.converters import proto_to_meeting_info +from noteflow.grpc.types import MeetingInfo, TranscriptSegment from noteflow.grpc.proto import noteflow_pb2 from noteflow.infrastructure.logging import get_client_rate_limiter, get_logger if TYPE_CHECKING: - from noteflow.grpc._client_mixins.protocols import ClientHost + from noteflow.grpc.client_mixins.protocols import ClientHost logger = get_logger(__name__) _rate_limiter = get_client_rate_limiter() diff --git a/src/noteflow/grpc/_client_mixins/protocols.py b/src/noteflow/grpc/client_mixins/protocols.py similarity index 96% rename from src/noteflow/grpc/_client_mixins/protocols.py rename to src/noteflow/grpc/client_mixins/protocols.py index 415ab04..b7af25f 100644 --- a/src/noteflow/grpc/_client_mixins/protocols.py +++ b/src/noteflow/grpc/client_mixins/protocols.py @@ -11,8 +11,8 @@ if TYPE_CHECKING: import numpy as np from numpy.typing import NDArray - from noteflow.grpc._client_mixins.converters import ProtoAnnotation, ProtoMeeting, ProtoSegment - from noteflow.grpc._types import ConnectionCallback, TranscriptCallback, TranscriptSegment + from noteflow.grpc.client_mixins.converters import ProtoAnnotation, ProtoMeeting, ProtoSegment + from noteflow.grpc.types import ConnectionCallback, TranscriptCallback, TranscriptSegment from noteflow.grpc.proto import noteflow_pb2 diff --git a/src/noteflow/grpc/_client_mixins/streaming.py b/src/noteflow/grpc/client_mixins/streaming.py similarity index 96% rename from src/noteflow/grpc/_client_mixins/streaming.py rename to src/noteflow/grpc/client_mixins/streaming.py index 70ba8bf..6d20c28 100644 --- a/src/noteflow/grpc/_client_mixins/streaming.py +++ b/src/noteflow/grpc/client_mixins/streaming.py @@ -11,8 +11,8 @@ from typing import TYPE_CHECKING import grpc from noteflow.config.constants import DEFAULT_SAMPLE_RATE -from noteflow.grpc._config import STREAMING_CONFIG -from noteflow.grpc._types import ConnectionCallback, TranscriptCallback, TranscriptSegment +from noteflow.grpc.config.config import STREAMING_CONFIG +from noteflow.grpc.types import ConnectionCallback, TranscriptCallback, TranscriptSegment from noteflow.grpc.proto import noteflow_pb2 from noteflow.infrastructure.logging import get_client_rate_limiter, get_logger @@ -20,7 +20,7 @@ if TYPE_CHECKING: import numpy as np from numpy.typing import NDArray - from noteflow.grpc._client_mixins.protocols import ClientHost, ProtoTranscriptUpdate + from noteflow.grpc.client_mixins.protocols import ClientHost, ProtoTranscriptUpdate logger = get_logger(__name__) _rate_limiter = get_client_rate_limiter() diff --git a/src/noteflow/grpc/config/__init__.py b/src/noteflow/grpc/config/__init__.py new file mode 100644 index 0000000..ec27f8d --- /dev/null +++ b/src/noteflow/grpc/config/__init__.py @@ -0,0 +1 @@ +"""Configuration helpers for gRPC server and client.""" diff --git a/src/noteflow/grpc/_cli.py b/src/noteflow/grpc/config/cli.py similarity index 99% rename from src/noteflow/grpc/_cli.py rename to src/noteflow/grpc/config/cli.py index c0bc59e..540a0a4 100644 --- a/src/noteflow/grpc/_cli.py +++ b/src/noteflow/grpc/config/cli.py @@ -10,7 +10,7 @@ from noteflow.config.constants import DEFAULT_GRPC_PORT from noteflow.infrastructure.asr.engine import VALID_MODEL_SIZES from noteflow.infrastructure.logging import get_logger -from ._config import ( +from .config import ( DEFAULT_BIND_ADDRESS, DEFAULT_MODEL, AsrConfig, diff --git a/src/noteflow/grpc/_config.py b/src/noteflow/grpc/config/config.py similarity index 97% rename from src/noteflow/grpc/_config.py rename to src/noteflow/grpc/config/config.py index 7870a1e..2bc8628 100644 --- a/src/noteflow/grpc/_config.py +++ b/src/noteflow/grpc/config/config.py @@ -10,10 +10,10 @@ from noteflow.config.constants import DEFAULT_GRPC_PORT if TYPE_CHECKING: from noteflow.application.services.calendar import CalendarService from noteflow.application.services.identity import IdentityService - from noteflow.application.services.ner_service import NerService + from noteflow.application.services.ner import NerService from noteflow.application.services.project_service import ProjectService from noteflow.application.services.summarization import SummarizationService - from noteflow.application.services.webhook_service import WebhookService + from noteflow.application.services.webhooks import WebhookService from noteflow.infrastructure.diarization import DiarizationEngine # ============================================================================= diff --git a/src/noteflow/grpc/_constants.py b/src/noteflow/grpc/constants.py similarity index 100% rename from src/noteflow/grpc/_constants.py rename to src/noteflow/grpc/constants.py diff --git a/src/noteflow/grpc/identity/__init__.py b/src/noteflow/grpc/identity/__init__.py new file mode 100644 index 0000000..4f33dfa --- /dev/null +++ b/src/noteflow/grpc/identity/__init__.py @@ -0,0 +1 @@ +"""Identity helpers for gRPC runtime.""" diff --git a/src/noteflow/grpc/_identity_singleton.py b/src/noteflow/grpc/identity/singleton.py similarity index 100% rename from src/noteflow/grpc/_identity_singleton.py rename to src/noteflow/grpc/identity/singleton.py diff --git a/src/noteflow/grpc/interceptors/identity.py b/src/noteflow/grpc/interceptors/identity.py index e861b37..5a98885 100644 --- a/src/noteflow/grpc/interceptors/identity.py +++ b/src/noteflow/grpc/interceptors/identity.py @@ -99,9 +99,6 @@ async def _reject_unary_stream( context: aio.ServicerContext[TRequest, TResponse], ) -> AsyncIterator[TResponse]: await context.abort(grpc.StatusCode.UNAUTHENTICATED, message) - if False: - # Unreachable; keeps async generator type for gRPC handler signature. - yield cast(TResponse, None) async def _reject_stream_unary( @@ -119,9 +116,6 @@ async def _reject_stream_stream( context: aio.ServicerContext[TRequest, TResponse], ) -> AsyncIterator[TResponse]: await context.abort(grpc.StatusCode.UNAUTHENTICATED, message) - if False: - # Unreachable; keeps async generator type for gRPC handler signature. - yield cast(TResponse, None) class IdentityInterceptor(aio.ServerInterceptor): diff --git a/src/noteflow/grpc/_mixins/__init__.py b/src/noteflow/grpc/mixins/__init__.py similarity index 100% rename from src/noteflow/grpc/_mixins/__init__.py rename to src/noteflow/grpc/mixins/__init__.py diff --git a/src/noteflow/grpc/_mixins/_audio_processing.py b/src/noteflow/grpc/mixins/_audio_processing.py similarity index 100% rename from src/noteflow/grpc/_mixins/_audio_processing.py rename to src/noteflow/grpc/mixins/_audio_processing.py diff --git a/src/noteflow/grpc/mixins/_metrics.py b/src/noteflow/grpc/mixins/_metrics.py new file mode 100644 index 0000000..9845e80 --- /dev/null +++ b/src/noteflow/grpc/mixins/_metrics.py @@ -0,0 +1,85 @@ +"""Streaming operation error metrics for observability. + +Provides lightweight tracking of streaming errors to aid in diagnostics +and health monitoring without blocking RPC operations. + +Sprint GAP-003: Error Handling Mismatches +""" + +from __future__ import annotations + +from collections import Counter +from dataclasses import dataclass, field +from threading import Lock + +from noteflow.infrastructure.logging import get_logger + +logger = get_logger(__name__) + + +@dataclass +class StreamingErrorMetrics: + """Track streaming operation errors for observability. + + Thread-safe counter for error types occurring during streaming operations. + Errors are recorded by type (e.g., "audio_decode_failed", "diarization_failed") + and meeting ID for correlation with logs. + """ + + _errors: Counter[str] = field(default_factory=Counter) + _lock: Lock = field(default_factory=Lock) + + def record_error(self, error_type: str, meeting_id: str) -> None: + """Record a streaming error occurrence. + + Args: + error_type: Type of error (e.g., "audio_decode_failed"). + meeting_id: Meeting ID where the error occurred. + """ + with self._lock: + self._errors[error_type] += 1 + logger.debug( + "streaming_error_recorded", + error_type=error_type, + meeting_id=meeting_id, + total_count=self._errors[error_type], + ) + + def get_counts(self) -> dict[str, int]: + """Get current error counts by type. + + Returns: + Dictionary mapping error type to occurrence count. + """ + with self._lock: + return dict(self._errors) + + def reset(self) -> None: + """Reset all error counts (primarily for testing).""" + with self._lock: + self._errors.clear() + logger.debug("streaming_error_metrics_reset") + + +# Global singleton +_streaming_metrics: StreamingErrorMetrics | None = None + + +def get_streaming_metrics() -> StreamingErrorMetrics: + """Get the global streaming metrics singleton. + + Returns: + Shared StreamingErrorMetrics instance. + """ + global _streaming_metrics + if _streaming_metrics is None: + _streaming_metrics = StreamingErrorMetrics() + return _streaming_metrics + + +def reset_streaming_metrics() -> None: + """Reset the global streaming metrics singleton (testing helper).""" + global _streaming_metrics + if _streaming_metrics is not None: + _streaming_metrics.reset() + _streaming_metrics = None diff --git a/src/noteflow/grpc/_mixins/_model_status.py b/src/noteflow/grpc/mixins/_model_status.py similarity index 93% rename from src/noteflow/grpc/_mixins/_model_status.py rename to src/noteflow/grpc/mixins/_model_status.py index 551c327..fa2a316 100644 --- a/src/noteflow/grpc/_mixins/_model_status.py +++ b/src/noteflow/grpc/mixins/_model_status.py @@ -7,7 +7,7 @@ from typing import TYPE_CHECKING, Protocol from noteflow.infrastructure.logging import get_logger if TYPE_CHECKING: - from noteflow.application.services.ner_service import NerService + from noteflow.application.services.ner import NerService from noteflow.application.services.summarization import SummarizationService from noteflow.infrastructure.asr import FasterWhisperEngine from noteflow.infrastructure.diarization.engine import DiarizationEngine @@ -92,9 +92,10 @@ def _append_summarization_status( payload["summarization_provider_count"] = len(service.providers) return - summaries: list[str] = [] - for mode, provider in service.providers.items(): - summaries.append(_format_provider_summary(mode, provider)) + summaries: list[str] = [ + _format_provider_summary(mode, provider) + for mode, provider in service.providers.items() + ] payload["summarization_providers"] = summaries @@ -115,9 +116,7 @@ def _format_provider_summary(mode: object, provider: SummarizerProvider) -> str: def _read_provider_model(provider: SummarizerProvider) -> str: raw_model = getattr(provider, "_model", None) - if isinstance(raw_model, str): - return raw_model - return "" + return raw_model if isinstance(raw_model, str) else "" def _read_provider_client_ready(provider: SummarizerProvider) -> bool: diff --git a/src/noteflow/grpc/_mixins/_repository_protocols.py b/src/noteflow/grpc/mixins/_repository_protocols.py similarity index 100% rename from src/noteflow/grpc/_mixins/_repository_protocols.py rename to src/noteflow/grpc/mixins/_repository_protocols.py diff --git a/src/noteflow/grpc/_mixins/_servicer_state.py b/src/noteflow/grpc/mixins/_servicer_state.py similarity index 91% rename from src/noteflow/grpc/_mixins/_servicer_state.py rename to src/noteflow/grpc/mixins/_servicer_state.py index dfbc3f2..6e2fe3c 100644 --- a/src/noteflow/grpc/_mixins/_servicer_state.py +++ b/src/noteflow/grpc/mixins/_servicer_state.py @@ -13,16 +13,16 @@ if TYPE_CHECKING: from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker - from noteflow.application.services.asr_config_service import AsrConfigService + from noteflow.application.services.asr_config import AsrConfigService from noteflow.application.services.calendar import CalendarService - from noteflow.application.services.hf_token_service import HfTokenService + from noteflow.application.services.huggingface import HfTokenService from noteflow.application.services.identity import IdentityService - from noteflow.application.services.ner_service import NerService + from noteflow.application.services.ner import NerService from noteflow.application.services.project_service import ProjectService from noteflow.application.services.summarization import SummarizationService - from noteflow.application.services.webhook_service import WebhookService + from noteflow.application.services.webhooks import WebhookService from noteflow.domain.entities import SyncRun - from noteflow.application.services.streaming_config_persistence import StreamingConfig + from noteflow.application.services.streaming_config import StreamingConfig from noteflow.infrastructure.asr import FasterWhisperEngine, Segmenter, StreamingVad from noteflow.infrastructure.audio.writer import MeetingAudioWriter from noteflow.infrastructure.auth.oidc_registry import OidcAuthService diff --git a/src/noteflow/grpc/mixins/_task_callbacks.py b/src/noteflow/grpc/mixins/_task_callbacks.py new file mode 100644 index 0000000..0ea5814 --- /dev/null +++ b/src/noteflow/grpc/mixins/_task_callbacks.py @@ -0,0 +1,78 @@ +"""Task done callback helpers for background job management. + +Provides reusable callback factories for asyncio task completion handling, +particularly for marking failed jobs in the database. + +Sprint GAP-003: Error Handling Mismatches +""" + +from __future__ import annotations + +import asyncio +from collections.abc import Awaitable, Callable + +from noteflow.infrastructure.logging import get_logger + +logger = get_logger(__name__) + + +def create_job_done_callback( + job_id: str, + tasks_dict: dict[str, asyncio.Task[None]], + mark_failed: Callable[[str, str], Awaitable[None]], +) -> Callable[[asyncio.Task[None]], None]: + """Create task done callback that marks jobs failed on exception. + + Background tasks that process jobs need to update job status when they + fail. Since task done callbacks are synchronous, this factory creates + a callback that schedules the async mark_failed coroutine. + + Args: + job_id: Identifier for the job being processed. + tasks_dict: Dictionary tracking active tasks (for cleanup). + mark_failed: Async function to mark job as failed (job_id, error_message). + + Returns: + Synchronous callback suitable for Task.add_done_callback. + + Example: + task = asyncio.create_task(process_job(job_id)) + callback = create_job_done_callback( + job_id, self.job_tasks, self._mark_job_failed + ) + task.add_done_callback(callback) + """ + + def callback(task: asyncio.Task[None]) -> None: + """Handle task completion, scheduling failure updates on exception.""" + tasks_dict.pop(job_id, None) + + if task.cancelled(): + logger.debug("job_task_cancelled", job_id=job_id) + return + + exc = task.exception() + if exc is None: + return + + # Log the exception + logger.exception( + "job_task_failed_unhandled", + job_id=job_id, + exc_info=exc, + ) + + # Schedule the mark_failed coroutine (fire-and-forget) + # Note: This requires an active event loop, which should exist + # since we're being called from asyncio task completion + try: + asyncio.create_task(mark_failed(job_id, str(exc))) + except RuntimeError as schedule_err: + # If we can't schedule (e.g., loop closed), log but don't crash + logger.error( + "Failed to schedule mark_failed for job %s: %s", + job_id, + schedule_err, + ) + + return callback diff --git a/src/noteflow/grpc/_mixins/_types.py b/src/noteflow/grpc/mixins/_types.py similarity index 100% rename from src/noteflow/grpc/_mixins/_types.py rename to src/noteflow/grpc/mixins/_types.py diff --git a/src/noteflow/grpc/_mixins/annotation.py b/src/noteflow/grpc/mixins/annotation.py similarity index 100% rename from src/noteflow/grpc/_mixins/annotation.py rename to src/noteflow/grpc/mixins/annotation.py diff --git a/src/noteflow/grpc/_mixins/asr_config.py b/src/noteflow/grpc/mixins/asr_config.py similarity index 99% rename from src/noteflow/grpc/_mixins/asr_config.py rename to src/noteflow/grpc/mixins/asr_config.py index 7d1ce9c..ef8e730 100644 --- a/src/noteflow/grpc/_mixins/asr_config.py +++ b/src/noteflow/grpc/mixins/asr_config.py @@ -8,7 +8,7 @@ from __future__ import annotations from typing import Protocol, cast -from noteflow.application.services.asr_config_service import ( +from noteflow.application.services.asr_config import ( AsrComputeType, AsrConfigPhase, AsrConfigService, diff --git a/src/noteflow/grpc/_mixins/calendar.py b/src/noteflow/grpc/mixins/calendar.py similarity index 100% rename from src/noteflow/grpc/_mixins/calendar.py rename to src/noteflow/grpc/mixins/calendar.py diff --git a/src/noteflow/grpc/_mixins/converters/__init__.py b/src/noteflow/grpc/mixins/converters/__init__.py similarity index 100% rename from src/noteflow/grpc/_mixins/converters/__init__.py rename to src/noteflow/grpc/mixins/converters/__init__.py diff --git a/src/noteflow/grpc/_mixins/converters/_domain.py b/src/noteflow/grpc/mixins/converters/_domain.py similarity index 100% rename from src/noteflow/grpc/_mixins/converters/_domain.py rename to src/noteflow/grpc/mixins/converters/_domain.py diff --git a/src/noteflow/grpc/_mixins/converters/_external.py b/src/noteflow/grpc/mixins/converters/_external.py similarity index 100% rename from src/noteflow/grpc/_mixins/converters/_external.py rename to src/noteflow/grpc/mixins/converters/_external.py diff --git a/src/noteflow/grpc/_mixins/converters/_id_parsing.py b/src/noteflow/grpc/mixins/converters/_id_parsing.py similarity index 100% rename from src/noteflow/grpc/_mixins/converters/_id_parsing.py rename to src/noteflow/grpc/mixins/converters/_id_parsing.py diff --git a/src/noteflow/grpc/_mixins/converters/_oidc.py b/src/noteflow/grpc/mixins/converters/_oidc.py similarity index 100% rename from src/noteflow/grpc/_mixins/converters/_oidc.py rename to src/noteflow/grpc/mixins/converters/_oidc.py diff --git a/src/noteflow/grpc/_mixins/converters/_timestamps.py b/src/noteflow/grpc/mixins/converters/_timestamps.py similarity index 100% rename from src/noteflow/grpc/_mixins/converters/_timestamps.py rename to src/noteflow/grpc/mixins/converters/_timestamps.py diff --git a/src/noteflow/grpc/_mixins/diarization/__init__.py b/src/noteflow/grpc/mixins/diarization/__init__.py similarity index 100% rename from src/noteflow/grpc/_mixins/diarization/__init__.py rename to src/noteflow/grpc/mixins/diarization/__init__.py diff --git a/src/noteflow/grpc/_mixins/diarization/_jobs.py b/src/noteflow/grpc/mixins/diarization/_jobs.py similarity index 100% rename from src/noteflow/grpc/_mixins/diarization/_jobs.py rename to src/noteflow/grpc/mixins/diarization/_jobs.py diff --git a/src/noteflow/grpc/_mixins/diarization/_mixin.py b/src/noteflow/grpc/mixins/diarization/_mixin.py similarity index 100% rename from src/noteflow/grpc/_mixins/diarization/_mixin.py rename to src/noteflow/grpc/mixins/diarization/_mixin.py diff --git a/src/noteflow/grpc/_mixins/diarization/_refinement.py b/src/noteflow/grpc/mixins/diarization/_refinement.py similarity index 100% rename from src/noteflow/grpc/_mixins/diarization/_refinement.py rename to src/noteflow/grpc/mixins/diarization/_refinement.py diff --git a/src/noteflow/grpc/_mixins/diarization/_speaker.py b/src/noteflow/grpc/mixins/diarization/_speaker.py similarity index 100% rename from src/noteflow/grpc/_mixins/diarization/_speaker.py rename to src/noteflow/grpc/mixins/diarization/_speaker.py diff --git a/src/noteflow/grpc/_mixins/diarization/_status.py b/src/noteflow/grpc/mixins/diarization/_status.py similarity index 100% rename from src/noteflow/grpc/_mixins/diarization/_status.py rename to src/noteflow/grpc/mixins/diarization/_status.py diff --git a/src/noteflow/grpc/_mixins/diarization/_streaming.py b/src/noteflow/grpc/mixins/diarization/_streaming.py similarity index 100% rename from src/noteflow/grpc/_mixins/diarization/_streaming.py rename to src/noteflow/grpc/mixins/diarization/_streaming.py diff --git a/src/noteflow/grpc/_mixins/diarization/_types.py b/src/noteflow/grpc/mixins/diarization/_types.py similarity index 100% rename from src/noteflow/grpc/_mixins/diarization/_types.py rename to src/noteflow/grpc/mixins/diarization/_types.py diff --git a/src/noteflow/grpc/_mixins/diarization_job.py b/src/noteflow/grpc/mixins/diarization_job.py similarity index 100% rename from src/noteflow/grpc/_mixins/diarization_job.py rename to src/noteflow/grpc/mixins/diarization_job.py diff --git a/src/noteflow/grpc/_mixins/entities.py b/src/noteflow/grpc/mixins/entities.py similarity index 98% rename from src/noteflow/grpc/_mixins/entities.py rename to src/noteflow/grpc/mixins/entities.py index 33e1347..efdac06 100644 --- a/src/noteflow/grpc/_mixins/entities.py +++ b/src/noteflow/grpc/mixins/entities.py @@ -25,7 +25,7 @@ from .protocols import EntitiesRepositoryProvider if TYPE_CHECKING: from collections.abc import Callable - from noteflow.application.services.ner_service import NerService + from noteflow.application.services.ner import NerService logger = get_logger(__name__) diff --git a/src/noteflow/grpc/_mixins/errors/__init__.py b/src/noteflow/grpc/mixins/errors/__init__.py similarity index 100% rename from src/noteflow/grpc/_mixins/errors/__init__.py rename to src/noteflow/grpc/mixins/errors/__init__.py diff --git a/src/noteflow/grpc/_mixins/errors/_abort.py b/src/noteflow/grpc/mixins/errors/_abort.py similarity index 100% rename from src/noteflow/grpc/_mixins/errors/_abort.py rename to src/noteflow/grpc/mixins/errors/_abort.py diff --git a/src/noteflow/grpc/_mixins/errors/_constants.py b/src/noteflow/grpc/mixins/errors/_constants.py similarity index 100% rename from src/noteflow/grpc/_mixins/errors/_constants.py rename to src/noteflow/grpc/mixins/errors/_constants.py diff --git a/src/noteflow/grpc/_mixins/errors/_fetch.py b/src/noteflow/grpc/mixins/errors/_fetch.py similarity index 100% rename from src/noteflow/grpc/_mixins/errors/_fetch.py rename to src/noteflow/grpc/mixins/errors/_fetch.py diff --git a/src/noteflow/grpc/_mixins/errors/_parse.py b/src/noteflow/grpc/mixins/errors/_parse.py similarity index 100% rename from src/noteflow/grpc/_mixins/errors/_parse.py rename to src/noteflow/grpc/mixins/errors/_parse.py diff --git a/src/noteflow/grpc/_mixins/errors/_require.py b/src/noteflow/grpc/mixins/errors/_require.py similarity index 98% rename from src/noteflow/grpc/_mixins/errors/_require.py rename to src/noteflow/grpc/mixins/errors/_require.py index 4f5192f..f662cc6 100644 --- a/src/noteflow/grpc/_mixins/errors/_require.py +++ b/src/noteflow/grpc/mixins/errors/_require.py @@ -10,7 +10,7 @@ from typing import TYPE_CHECKING, Protocol from noteflow.config.constants import FEATURE_NAME_PROJECTS -from ..._constants import WORKSPACES_LABEL +from ...constants import WORKSPACES_LABEL from ._abort import ( AbortableContext, abort_database_required, @@ -19,7 +19,7 @@ from ._abort import ( ) if TYPE_CHECKING: - from noteflow.application.services.ner_service import NerService + from noteflow.application.services.ner import NerService from noteflow.application.services.project_service import ProjectService diff --git a/src/noteflow/grpc/_mixins/errors/_webhooks.py b/src/noteflow/grpc/mixins/errors/_webhooks.py similarity index 100% rename from src/noteflow/grpc/_mixins/errors/_webhooks.py rename to src/noteflow/grpc/mixins/errors/_webhooks.py diff --git a/src/noteflow/grpc/_mixins/export.py b/src/noteflow/grpc/mixins/export.py similarity index 97% rename from src/noteflow/grpc/_mixins/export.py rename to src/noteflow/grpc/mixins/export.py index 1d728b8..6fb0ddb 100644 --- a/src/noteflow/grpc/_mixins/export.py +++ b/src/noteflow/grpc/mixins/export.py @@ -5,7 +5,7 @@ from __future__ import annotations import base64 from typing import TYPE_CHECKING, cast -from noteflow.application.services.export_service import ExportFormat, ExportService +from noteflow.application.services.export import ExportFormat, ExportService from noteflow.application.services.protocols import ExportRepositoryProvider from noteflow.config.constants import EXPORT_EXT_HTML, EXPORT_EXT_PDF, EXPORT_FORMAT_HTML from noteflow.config.constants.encoding import ASCII_ENCODING diff --git a/src/noteflow/grpc/_mixins/hf_token.py b/src/noteflow/grpc/mixins/hf_token.py similarity index 98% rename from src/noteflow/grpc/_mixins/hf_token.py rename to src/noteflow/grpc/mixins/hf_token.py index 1401a75..2c1cf74 100644 --- a/src/noteflow/grpc/_mixins/hf_token.py +++ b/src/noteflow/grpc/mixins/hf_token.py @@ -14,7 +14,7 @@ from ..proto import noteflow_pb2 from ._types import GrpcContext if TYPE_CHECKING: - from noteflow.application.services.hf_token_service import HfTokenService + from noteflow.application.services.huggingface import HfTokenService logger = get_logger(__name__) diff --git a/src/noteflow/grpc/_mixins/identity.py b/src/noteflow/grpc/mixins/identity.py similarity index 99% rename from src/noteflow/grpc/_mixins/identity.py rename to src/noteflow/grpc/mixins/identity.py index 3a77154..4f9571e 100644 --- a/src/noteflow/grpc/_mixins/identity.py +++ b/src/noteflow/grpc/mixins/identity.py @@ -11,7 +11,7 @@ from noteflow.domain.entities.integration import IntegrationType from noteflow.domain.ports.unit_of_work import UnitOfWork from noteflow.infrastructure.logging import get_logger -from .._constants import WORKSPACES_LABEL +from ..constants import WORKSPACES_LABEL from ..proto import noteflow_pb2 from ._types import GrpcContext from .errors import ( diff --git a/src/noteflow/grpc/_mixins/meeting/__init__.py b/src/noteflow/grpc/mixins/meeting/__init__.py similarity index 100% rename from src/noteflow/grpc/_mixins/meeting/__init__.py rename to src/noteflow/grpc/mixins/meeting/__init__.py diff --git a/src/noteflow/grpc/_mixins/meeting/_post_processing.py b/src/noteflow/grpc/mixins/meeting/_post_processing.py similarity index 98% rename from src/noteflow/grpc/_mixins/meeting/_post_processing.py rename to src/noteflow/grpc/mixins/meeting/_post_processing.py index 7d75cc9..4a5c9e2 100644 --- a/src/noteflow/grpc/_mixins/meeting/_post_processing.py +++ b/src/noteflow/grpc/mixins/meeting/_post_processing.py @@ -13,7 +13,7 @@ from noteflow.infrastructure.logging import get_logger, log_state_transition if TYPE_CHECKING: from noteflow.application.services.summarization import SummarizationService - from noteflow.application.services.webhook_service import WebhookService + from noteflow.application.services.webhooks import WebhookService from noteflow.domain.entities import Meeting, Segment, Summary from ..protocols import ServicerHost diff --git a/src/noteflow/grpc/_mixins/meeting/_project_scope.py b/src/noteflow/grpc/mixins/meeting/_project_scope.py similarity index 96% rename from src/noteflow/grpc/_mixins/meeting/_project_scope.py rename to src/noteflow/grpc/mixins/meeting/_project_scope.py index e0724cc..ce3b0ef 100644 --- a/src/noteflow/grpc/_mixins/meeting/_project_scope.py +++ b/src/noteflow/grpc/mixins/meeting/_project_scope.py @@ -35,7 +35,7 @@ async def parse_project_ids_or_abort( for raw_project_id in request.project_ids: try: project_ids.append(UUID(raw_project_id)) - except ValueError: + except ValueError as e: truncated = ( f"{raw_project_id[:ID_TRUNCATE_LEN]}..." if len(raw_project_id) > ID_TRUNCATE_LEN @@ -50,7 +50,7 @@ async def parse_project_ids_or_abort( context, f"{ERROR_INVALID_PROJECT_ID_PREFIX}{raw_project_id}", ) - raise AssertionError("unreachable") # abort is NoReturn + raise AssertionError("unreachable") from e return project_ids @@ -65,7 +65,7 @@ async def parse_project_id_or_abort( try: return UUID(request.project_id) - except ValueError: + except ValueError as e: truncated = ( f"{request.project_id[:ID_TRUNCATE_LEN]}..." if len(request.project_id) > ID_TRUNCATE_LEN @@ -78,7 +78,7 @@ async def parse_project_id_or_abort( ) error_message = f"{ERROR_INVALID_PROJECT_ID_PREFIX}{request.project_id}" await abort_invalid_argument(context, error_message) - raise AssertionError("unreachable") # abort is NoReturn + raise AssertionError("unreachable") from e async def resolve_active_project_id( diff --git a/src/noteflow/grpc/_mixins/meeting/_stop_ops.py b/src/noteflow/grpc/mixins/meeting/_stop_ops.py similarity index 97% rename from src/noteflow/grpc/_mixins/meeting/_stop_ops.py rename to src/noteflow/grpc/mixins/meeting/_stop_ops.py index b74fc98..fb84e12 100644 --- a/src/noteflow/grpc/_mixins/meeting/_stop_ops.py +++ b/src/noteflow/grpc/mixins/meeting/_stop_ops.py @@ -16,7 +16,7 @@ from ...proto import noteflow_pb2 from ..errors import abort_invalid_argument, parse_optional_uuid_or_abort if TYPE_CHECKING: - from noteflow.application.services.webhook_service import WebhookService + from noteflow.application.services.webhooks import WebhookService from noteflow.infrastructure.audio.writer import MeetingAudioWriter from .._types import GrpcContext diff --git a/src/noteflow/grpc/_mixins/meeting/meeting_mixin.py b/src/noteflow/grpc/mixins/meeting/meeting_mixin.py similarity index 99% rename from src/noteflow/grpc/_mixins/meeting/meeting_mixin.py rename to src/noteflow/grpc/mixins/meeting/meeting_mixin.py index 634eb71..470caaa 100644 --- a/src/noteflow/grpc/_mixins/meeting/meeting_mixin.py +++ b/src/noteflow/grpc/mixins/meeting/meeting_mixin.py @@ -37,7 +37,7 @@ if TYPE_CHECKING: from collections.abc import Callable from noteflow.application.services.project_service import ProjectService - from noteflow.application.services.webhook_service import WebhookService + from noteflow.application.services.webhooks import WebhookService from noteflow.infrastructure.audio.writer import MeetingAudioWriter from .._types import GrpcContext diff --git a/src/noteflow/grpc/_mixins/observability.py b/src/noteflow/grpc/mixins/observability.py similarity index 100% rename from src/noteflow/grpc/_mixins/observability.py rename to src/noteflow/grpc/mixins/observability.py diff --git a/src/noteflow/grpc/_mixins/oidc/__init__.py b/src/noteflow/grpc/mixins/oidc/__init__.py similarity index 100% rename from src/noteflow/grpc/_mixins/oidc/__init__.py rename to src/noteflow/grpc/mixins/oidc/__init__.py diff --git a/src/noteflow/grpc/_mixins/oidc/_support.py b/src/noteflow/grpc/mixins/oidc/_support.py similarity index 100% rename from src/noteflow/grpc/_mixins/oidc/_support.py rename to src/noteflow/grpc/mixins/oidc/_support.py diff --git a/src/noteflow/grpc/_mixins/oidc/oidc_mixin.py b/src/noteflow/grpc/mixins/oidc/oidc_mixin.py similarity index 100% rename from src/noteflow/grpc/_mixins/oidc/oidc_mixin.py rename to src/noteflow/grpc/mixins/oidc/oidc_mixin.py diff --git a/src/noteflow/grpc/_mixins/preferences.py b/src/noteflow/grpc/mixins/preferences.py similarity index 100% rename from src/noteflow/grpc/_mixins/preferences.py rename to src/noteflow/grpc/mixins/preferences.py diff --git a/src/noteflow/grpc/_mixins/project/__init__.py b/src/noteflow/grpc/mixins/project/__init__.py similarity index 100% rename from src/noteflow/grpc/_mixins/project/__init__.py rename to src/noteflow/grpc/mixins/project/__init__.py diff --git a/src/noteflow/grpc/_mixins/project/_converters.py b/src/noteflow/grpc/mixins/project/_converters.py similarity index 100% rename from src/noteflow/grpc/_mixins/project/_converters.py rename to src/noteflow/grpc/mixins/project/_converters.py diff --git a/src/noteflow/grpc/_mixins/project/_membership.py b/src/noteflow/grpc/mixins/project/_membership.py similarity index 100% rename from src/noteflow/grpc/_mixins/project/_membership.py rename to src/noteflow/grpc/mixins/project/_membership.py diff --git a/src/noteflow/grpc/_mixins/project/_mixin.py b/src/noteflow/grpc/mixins/project/_mixin.py similarity index 100% rename from src/noteflow/grpc/_mixins/project/_mixin.py rename to src/noteflow/grpc/mixins/project/_mixin.py diff --git a/src/noteflow/grpc/_mixins/protocols.py b/src/noteflow/grpc/mixins/protocols.py similarity index 100% rename from src/noteflow/grpc/_mixins/protocols.py rename to src/noteflow/grpc/mixins/protocols.py diff --git a/src/noteflow/grpc/_mixins/servicer_core/__init__.py b/src/noteflow/grpc/mixins/servicer_core/__init__.py similarity index 100% rename from src/noteflow/grpc/_mixins/servicer_core/__init__.py rename to src/noteflow/grpc/mixins/servicer_core/__init__.py diff --git a/src/noteflow/grpc/_mixins/servicer_core/protocols.py b/src/noteflow/grpc/mixins/servicer_core/protocols.py similarity index 100% rename from src/noteflow/grpc/_mixins/servicer_core/protocols.py rename to src/noteflow/grpc/mixins/servicer_core/protocols.py diff --git a/src/noteflow/grpc/_mixins/servicer_other/__init__.py b/src/noteflow/grpc/mixins/servicer_other/__init__.py similarity index 100% rename from src/noteflow/grpc/_mixins/servicer_other/__init__.py rename to src/noteflow/grpc/mixins/servicer_other/__init__.py diff --git a/src/noteflow/grpc/_mixins/servicer_other/protocols.py b/src/noteflow/grpc/mixins/servicer_other/protocols.py similarity index 98% rename from src/noteflow/grpc/_mixins/servicer_other/protocols.py rename to src/noteflow/grpc/mixins/servicer_other/protocols.py index 91a39e6..285137f 100644 --- a/src/noteflow/grpc/_mixins/servicer_other/protocols.py +++ b/src/noteflow/grpc/mixins/servicer_other/protocols.py @@ -11,7 +11,7 @@ if TYPE_CHECKING: from noteflow.domain.entities import Integration, Meeting, Segment, Summary, SyncRun from noteflow.domain.ports.unit_of_work import UnitOfWork from noteflow.domain.value_objects import MeetingId - from noteflow.grpc._mixins.preferences import PreferencesRepositoryProvider + from noteflow.grpc.mixins.preferences import PreferencesRepositoryProvider from noteflow.infrastructure.persistence.repositories.preferences_repo import ( PreferenceWithMetadata, ) diff --git a/src/noteflow/grpc/_mixins/streaming/__init__.py b/src/noteflow/grpc/mixins/streaming/__init__.py similarity index 100% rename from src/noteflow/grpc/_mixins/streaming/__init__.py rename to src/noteflow/grpc/mixins/streaming/__init__.py diff --git a/src/noteflow/grpc/_mixins/streaming/_asr.py b/src/noteflow/grpc/mixins/streaming/_asr.py similarity index 100% rename from src/noteflow/grpc/_mixins/streaming/_asr.py rename to src/noteflow/grpc/mixins/streaming/_asr.py diff --git a/src/noteflow/grpc/_mixins/streaming/_cleanup.py b/src/noteflow/grpc/mixins/streaming/_cleanup.py similarity index 100% rename from src/noteflow/grpc/_mixins/streaming/_cleanup.py rename to src/noteflow/grpc/mixins/streaming/_cleanup.py diff --git a/src/noteflow/grpc/_mixins/streaming/_mixin.py b/src/noteflow/grpc/mixins/streaming/_mixin.py similarity index 99% rename from src/noteflow/grpc/_mixins/streaming/_mixin.py rename to src/noteflow/grpc/mixins/streaming/_mixin.py index 5569378..8816356 100644 --- a/src/noteflow/grpc/_mixins/streaming/_mixin.py +++ b/src/noteflow/grpc/mixins/streaming/_mixin.py @@ -106,8 +106,7 @@ class StreamingMixin: ): yield update finally: - cleanup_meeting = stream_state.current or stream_state.initialized - if cleanup_meeting: + if cleanup_meeting := stream_state.current or stream_state.initialized: cleanup_stream_resources(self, cleanup_meeting) log_memory_snapshot("stream_cleanup_end", meeting_id=cleanup_meeting) diff --git a/src/noteflow/grpc/_mixins/streaming/_partials.py b/src/noteflow/grpc/mixins/streaming/_partials.py similarity index 100% rename from src/noteflow/grpc/_mixins/streaming/_partials.py rename to src/noteflow/grpc/mixins/streaming/_partials.py diff --git a/src/noteflow/grpc/_mixins/streaming/_processing/__init__.py b/src/noteflow/grpc/mixins/streaming/_processing/__init__.py similarity index 100% rename from src/noteflow/grpc/_mixins/streaming/_processing/__init__.py rename to src/noteflow/grpc/mixins/streaming/_processing/__init__.py diff --git a/src/noteflow/grpc/_mixins/streaming/_processing/_audio_ops.py b/src/noteflow/grpc/mixins/streaming/_processing/_audio_ops.py similarity index 100% rename from src/noteflow/grpc/_mixins/streaming/_processing/_audio_ops.py rename to src/noteflow/grpc/mixins/streaming/_processing/_audio_ops.py diff --git a/src/noteflow/grpc/_mixins/streaming/_processing/_chunk_tracking.py b/src/noteflow/grpc/mixins/streaming/_processing/_chunk_tracking.py similarity index 100% rename from src/noteflow/grpc/_mixins/streaming/_processing/_chunk_tracking.py rename to src/noteflow/grpc/mixins/streaming/_processing/_chunk_tracking.py diff --git a/src/noteflow/grpc/_mixins/streaming/_processing/_congestion.py b/src/noteflow/grpc/mixins/streaming/_processing/_congestion.py similarity index 96% rename from src/noteflow/grpc/_mixins/streaming/_processing/_congestion.py rename to src/noteflow/grpc/mixins/streaming/_processing/_congestion.py index a501d2f..302cf91 100644 --- a/src/noteflow/grpc/_mixins/streaming/_processing/_congestion.py +++ b/src/noteflow/grpc/mixins/streaming/_processing/_congestion.py @@ -28,8 +28,7 @@ def calculate_congestion_info( CongestionInfo with processing delay, queue depth, and throttle recommendation. """ processing_delay_ms = 0 - receipt_times = host.chunk_receipt_times.get(meeting_id) - if receipt_times: + if receipt_times := host.chunk_receipt_times.get(meeting_id): try: # Access [0] can race with popleft() in decrement_pending_chunks oldest_receipt = receipt_times[0] diff --git a/src/noteflow/grpc/_mixins/streaming/_processing/_constants.py b/src/noteflow/grpc/mixins/streaming/_processing/_constants.py similarity index 100% rename from src/noteflow/grpc/_mixins/streaming/_processing/_constants.py rename to src/noteflow/grpc/mixins/streaming/_processing/_constants.py diff --git a/src/noteflow/grpc/_mixins/streaming/_processing/_types.py b/src/noteflow/grpc/mixins/streaming/_processing/_types.py similarity index 100% rename from src/noteflow/grpc/_mixins/streaming/_processing/_types.py rename to src/noteflow/grpc/mixins/streaming/_processing/_types.py diff --git a/src/noteflow/grpc/_mixins/streaming/_processing/_vad_processing.py b/src/noteflow/grpc/mixins/streaming/_processing/_vad_processing.py similarity index 100% rename from src/noteflow/grpc/_mixins/streaming/_processing/_vad_processing.py rename to src/noteflow/grpc/mixins/streaming/_processing/_vad_processing.py diff --git a/src/noteflow/grpc/_mixins/streaming/_session.py b/src/noteflow/grpc/mixins/streaming/_session.py similarity index 98% rename from src/noteflow/grpc/_mixins/streaming/_session.py rename to src/noteflow/grpc/mixins/streaming/_session.py index 975d04a..454026d 100644 --- a/src/noteflow/grpc/_mixins/streaming/_session.py +++ b/src/noteflow/grpc/mixins/streaming/_session.py @@ -14,6 +14,7 @@ from noteflow.config.constants import ( ERROR_MSG_MEETING_PREFIX, STREAM_INIT_LOCK_TIMEOUT_SECONDS, ) +from noteflow.domain.constants.fields import STATE from noteflow.infrastructure.logging import get_logger from .._types import GrpcContext @@ -205,7 +206,7 @@ class StreamSessionManager: meeting_id, context, ) - except TimeoutError: + except TimeoutError as e: logger.error( "Stream initialization lock timeout for meeting %s after %.1fs", meeting_id, @@ -214,7 +215,7 @@ class StreamSessionManager: await abort_failed_precondition( context, "Stream initialization timed out - server may be overloaded" ) - raise AssertionError("unreachable") # abort is NoReturn + raise AssertionError("unreachable") from e return reserved @staticmethod @@ -233,7 +234,7 @@ class StreamSessionManager: logger.warning( "Clearing stale active stream for meeting %s (state=%s)", meeting_id, - getattr(meeting, "state", None), + getattr(meeting, STATE, None), ) host.active_streams.discard(meeting_id) host.active_streams.add(meeting_id) diff --git a/src/noteflow/grpc/_mixins/streaming/_session_helpers.py b/src/noteflow/grpc/mixins/streaming/_session_helpers.py similarity index 100% rename from src/noteflow/grpc/_mixins/streaming/_session_helpers.py rename to src/noteflow/grpc/mixins/streaming/_session_helpers.py diff --git a/src/noteflow/grpc/_mixins/streaming/_types.py b/src/noteflow/grpc/mixins/streaming/_types.py similarity index 100% rename from src/noteflow/grpc/_mixins/streaming/_types.py rename to src/noteflow/grpc/mixins/streaming/_types.py diff --git a/src/noteflow/grpc/_mixins/streaming_config.py b/src/noteflow/grpc/mixins/streaming_config.py similarity index 95% rename from src/noteflow/grpc/_mixins/streaming_config.py rename to src/noteflow/grpc/mixins/streaming_config.py index ca82aa9..71de775 100644 --- a/src/noteflow/grpc/_mixins/streaming_config.py +++ b/src/noteflow/grpc/mixins/streaming_config.py @@ -8,7 +8,7 @@ from __future__ import annotations from dataclasses import replace from typing import TYPE_CHECKING -from noteflow.application.services.streaming_config_persistence import ( +from noteflow.application.services.streaming_config import ( STREAMING_CONFIG_KEYS, STREAMING_CONFIG_RANGES, StreamingConfig, @@ -44,12 +44,11 @@ def _build_configuration_proto(config: StreamingConfig) -> noteflow_pb2.Streamin def _parse_update_request( request: noteflow_pb2.UpdateStreamingConfigurationRequest, ) -> dict[str, float]: - updates: dict[str, float] = {} - - for field in STREAMING_CONFIG_KEYS: - if request.HasField(field): - updates[field] = getattr(request, field) - + updates: dict[str, float] = { + field: getattr(request, field) + for field in STREAMING_CONFIG_KEYS + if request.HasField(field) + } return updates diff --git a/src/noteflow/grpc/_mixins/summarization/__init__.py b/src/noteflow/grpc/mixins/summarization/__init__.py similarity index 100% rename from src/noteflow/grpc/_mixins/summarization/__init__.py rename to src/noteflow/grpc/mixins/summarization/__init__.py diff --git a/src/noteflow/grpc/_mixins/summarization/_consent.py b/src/noteflow/grpc/mixins/summarization/_consent.py similarity index 100% rename from src/noteflow/grpc/_mixins/summarization/_consent.py rename to src/noteflow/grpc/mixins/summarization/_consent.py diff --git a/src/noteflow/grpc/_mixins/summarization/_consent_mixin.py b/src/noteflow/grpc/mixins/summarization/_consent_mixin.py similarity index 100% rename from src/noteflow/grpc/_mixins/summarization/_consent_mixin.py rename to src/noteflow/grpc/mixins/summarization/_consent_mixin.py diff --git a/src/noteflow/grpc/_mixins/summarization/_constants.py b/src/noteflow/grpc/mixins/summarization/_constants.py similarity index 100% rename from src/noteflow/grpc/_mixins/summarization/_constants.py rename to src/noteflow/grpc/mixins/summarization/_constants.py diff --git a/src/noteflow/grpc/_mixins/summarization/_context_builders.py b/src/noteflow/grpc/mixins/summarization/_context_builders.py similarity index 100% rename from src/noteflow/grpc/_mixins/summarization/_context_builders.py rename to src/noteflow/grpc/mixins/summarization/_context_builders.py diff --git a/src/noteflow/grpc/_mixins/summarization/_generation_mixin.py b/src/noteflow/grpc/mixins/summarization/_generation_mixin.py similarity index 98% rename from src/noteflow/grpc/_mixins/summarization/_generation_mixin.py rename to src/noteflow/grpc/mixins/summarization/_generation_mixin.py index 7635798..bb44d0a 100644 --- a/src/noteflow/grpc/_mixins/summarization/_generation_mixin.py +++ b/src/noteflow/grpc/mixins/summarization/_generation_mixin.py @@ -12,7 +12,7 @@ from noteflow.domain.value_objects import MeetingId from noteflow.infrastructure.logging import get_logger from ...proto import noteflow_pb2 -from ..._startup import auto_enable_cloud_llm +from ...startup.startup import auto_enable_cloud_llm from .._types import GrpcContext from .._model_status import log_model_status from ..converters import parse_meeting_id_or_abort, summary_to_proto @@ -28,7 +28,7 @@ if TYPE_CHECKING: from collections.abc import Callable from noteflow.application.services.summarization import SummarizationService - from noteflow.application.services.webhook_service import WebhookService + from noteflow.application.services.webhooks import WebhookService from noteflow.domain.entities import Meeting from noteflow.domain.identity import OperationContext diff --git a/src/noteflow/grpc/_mixins/summarization/_summary_generation.py b/src/noteflow/grpc/mixins/summarization/_summary_generation.py similarity index 100% rename from src/noteflow/grpc/_mixins/summarization/_summary_generation.py rename to src/noteflow/grpc/mixins/summarization/_summary_generation.py diff --git a/src/noteflow/grpc/_mixins/summarization/_template_crud.py b/src/noteflow/grpc/mixins/summarization/_template_crud.py similarity index 100% rename from src/noteflow/grpc/_mixins/summarization/_template_crud.py rename to src/noteflow/grpc/mixins/summarization/_template_crud.py diff --git a/src/noteflow/grpc/_mixins/summarization/_template_resolution.py b/src/noteflow/grpc/mixins/summarization/_template_resolution.py similarity index 100% rename from src/noteflow/grpc/_mixins/summarization/_template_resolution.py rename to src/noteflow/grpc/mixins/summarization/_template_resolution.py diff --git a/src/noteflow/grpc/_mixins/summarization/_templates_mixin.py b/src/noteflow/grpc/mixins/summarization/_templates_mixin.py similarity index 100% rename from src/noteflow/grpc/_mixins/summarization/_templates_mixin.py rename to src/noteflow/grpc/mixins/summarization/_templates_mixin.py diff --git a/src/noteflow/grpc/_mixins/sync.py b/src/noteflow/grpc/mixins/sync.py similarity index 100% rename from src/noteflow/grpc/_mixins/sync.py rename to src/noteflow/grpc/mixins/sync.py diff --git a/src/noteflow/grpc/_mixins/webhooks.py b/src/noteflow/grpc/mixins/webhooks.py similarity index 100% rename from src/noteflow/grpc/_mixins/webhooks.py rename to src/noteflow/grpc/mixins/webhooks.py diff --git a/src/noteflow/grpc/proto/noteflow_pb2_grpc.py b/src/noteflow/grpc/proto/noteflow_pb2_grpc.py index 5dfa573..8e85da5 100644 --- a/src/noteflow/grpc/proto/noteflow_pb2_grpc.py +++ b/src/noteflow/grpc/proto/noteflow_pb2_grpc.py @@ -1,14 +1,15 @@ # Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT! """Client and server classes corresponding to protobuf-defined services.""" + import grpc import warnings from . import noteflow_pb2 as noteflow__pb2 -GRPC_GENERATED_VERSION = '1.76.0' GRPC_VERSION = grpc.__version__ _version_not_supported = False +GRPC_GENERATED_VERSION = '1.76.0' try: from grpc._utilities import first_version_is_lower _version_not_supported = first_version_is_lower(GRPC_VERSION, GRPC_GENERATED_VERSION) @@ -17,8 +18,7 @@ except ImportError: if _version_not_supported: raise RuntimeError( - f'The grpc package installed is at version {GRPC_VERSION},' - + ' but the generated code in noteflow_pb2_grpc.py depends on' + f'The grpc package installed is at version {GRPC_VERSION}, but the generated code in noteflow_pb2_grpc.py depends on' + f' grpcio>={GRPC_GENERATED_VERSION}.' + f' Please upgrade your grpc module to grpcio>={GRPC_GENERATED_VERSION}' + f' or downgrade your generated code using grpcio-tools<={GRPC_VERSION}.' diff --git a/src/noteflow/grpc/server/__init__.py b/src/noteflow/grpc/server/__init__.py index 6bc720c..b7e49e9 100644 --- a/src/noteflow/grpc/server/__init__.py +++ b/src/noteflow/grpc/server/__init__.py @@ -10,10 +10,10 @@ from typing import TYPE_CHECKING, Unpack import grpc.aio from pydantic import ValidationError -from noteflow.application.services.asr_config_persistence import ( +from noteflow.application.services.asr_config import ( resolve_asr_config_preference, ) -from noteflow.application.services.streaming_config_persistence import ( +from noteflow.application.services.streaming_config import ( StreamingConfig, build_default_streaming_config, ) @@ -25,32 +25,32 @@ from noteflow.config.constants.core import MAIN_MODULE_NAME from noteflow.config.settings import get_settings from noteflow.infrastructure.logging import LoggingConfig, configure_logging, get_logger -from .._cli import build_config_from_args, parse_args -from .._config import DEFAULT_BIND_ADDRESS, AsrConfig, GrpcServerConfig, ServicesConfig -from .._startup import StartupServices -from .._startup_banner import print_startup_banner +from ..config.cli import build_config_from_args, parse_args +from ..config.config import DEFAULT_BIND_ADDRESS, AsrConfig, GrpcServerConfig, ServicesConfig +from ..startup.startup import StartupServices +from ..startup.banner import print_startup_banner from ..service import NoteFlowServicer -from ._bootstrap import ( +from .internal.bootstrap import ( create_services, init_db, recover_orphaned_jobs, register_shutdown_handlers, warn_webhooks_without_db, ) -from ._lifecycle import load_asr_engine, stop_server, warm_diarization_engine -from ._services import build_servicer, ensure_services -from ._setup import bind_server, create_server -from ._streaming_config import load_streaming_config_from_preferences -from ._types import ServerInitKwargs +from .internal.lifecycle import load_asr_engine, stop_server, warm_diarization_engine +from .internal.services import build_servicer, ensure_services +from .internal.setup import bind_server, create_server +from .internal.streaming_config import load_streaming_config_from_preferences +from .internal.types import ServerInitKwargs if TYPE_CHECKING: from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker from noteflow.application.services.calendar import CalendarService - from noteflow.application.services.ner_service import NerService + from noteflow.application.services.ner import NerService from noteflow.application.services.project_service import ProjectService from noteflow.application.services.summarization import SummarizationService - from noteflow.application.services.webhook_service import WebhookService + from noteflow.application.services.webhooks import WebhookService from noteflow.infrastructure.diarization import DiarizationEngine diff --git a/src/noteflow/grpc/server/health.py b/src/noteflow/grpc/server/health.py new file mode 100644 index 0000000..06b97da --- /dev/null +++ b/src/noteflow/grpc/server/health.py @@ -0,0 +1,64 @@ +"""Server health tracking utilities.""" + +from __future__ import annotations + +from dataclasses import dataclass + + +@dataclass(slots=True) +class ServerHealthState: + """Tracks server bootstrap health state.""" + + job_recovery_success: bool = True + consent_loading_success: bool = True + asr_config_loading_success: bool = True + job_recovery_error: str = "" + consent_loading_error: str = "" + asr_config_loading_error: str = "" + + @property + def is_healthy(self) -> bool: + """Return True when all health checks succeed.""" + return ( + self.job_recovery_success + and self.consent_loading_success + and self.asr_config_loading_success + ) + + def get_error_summary(self) -> str: + """Return a semicolon-delimited summary of failed checks.""" + if self.is_healthy: + return "" + + errors: list[str] = [] + if not self.job_recovery_success: + errors.append(self._format_error("job_recovery", self.job_recovery_error)) + if not self.consent_loading_success: + errors.append(self._format_error("consent_loading", self.consent_loading_error)) + if not self.asr_config_loading_success: + errors.append( + self._format_error("asr_config_loading", self.asr_config_loading_error) + ) + return "; ".join(errors) + + @staticmethod + def _format_error(key: str, message: str) -> str: + """Format a single error message entry.""" + return f"{key}: {message}" if message else f"{key}:" + + +_server_health: ServerHealthState | None = None + + +def get_server_health() -> ServerHealthState: + """Return the global ServerHealthState singleton.""" + global _server_health + if _server_health is None: + _server_health = ServerHealthState() + return _server_health + + +def reset_server_health() -> None: + """Reset the global ServerHealthState singleton.""" + global _server_health + _server_health = None diff --git a/src/noteflow/grpc/server/internal/__init__.py b/src/noteflow/grpc/server/internal/__init__.py new file mode 100644 index 0000000..d286f56 --- /dev/null +++ b/src/noteflow/grpc/server/internal/__init__.py @@ -0,0 +1 @@ +"""Internal helpers for gRPC server startup.""" diff --git a/src/noteflow/grpc/server/_bootstrap.py b/src/noteflow/grpc/server/internal/bootstrap.py similarity index 92% rename from src/noteflow/grpc/server/_bootstrap.py rename to src/noteflow/grpc/server/internal/bootstrap.py index b1186fe..a9122a4 100644 --- a/src/noteflow/grpc/server/_bootstrap.py +++ b/src/noteflow/grpc/server/internal/bootstrap.py @@ -9,10 +9,10 @@ from typing import TYPE_CHECKING from noteflow.config.settings import get_feature_flags, get_settings from noteflow.infrastructure.logging import get_logger -from .._config import GrpcServerConfig, ServicesConfig -from .._server_bootstrap import mark_orphaned_jobs_failed -from .._startup import init_database_and_recovery, setup_summarization_with_consent -from .._startup_services import ( +from ...config.config import GrpcServerConfig, ServicesConfig +from ...startup.server_bootstrap import mark_orphaned_jobs_failed +from ...startup.startup import init_database_and_recovery, setup_summarization_with_consent +from ...startup.services import ( create_calendar_service, create_diarization_engine, create_ner_service, @@ -89,7 +89,7 @@ def register_shutdown_handlers(loop: asyncio.AbstractEventLoop) -> asyncio.Event for sig in (signal.SIGINT, signal.SIGTERM): try: loop.add_signal_handler(sig, signal_handler) - except (NotImplementedError, RuntimeError, ValueError) as exc: + except (RuntimeError, ValueError) as exc: logger.warning( "Signal handlers not supported; relying on default handlers", signal=sig, diff --git a/src/noteflow/grpc/server/_lifecycle.py b/src/noteflow/grpc/server/internal/lifecycle.py similarity index 98% rename from src/noteflow/grpc/server/_lifecycle.py rename to src/noteflow/grpc/server/internal/lifecycle.py index f7bde9d..f5b7c49 100644 --- a/src/noteflow/grpc/server/_lifecycle.py +++ b/src/noteflow/grpc/server/internal/lifecycle.py @@ -11,7 +11,7 @@ from noteflow.infrastructure.asr import FasterWhisperEngine from noteflow.infrastructure.logging import get_logger from sqlalchemy.ext.asyncio import AsyncEngine -from ..service import NoteFlowServicer +from ...service import NoteFlowServicer if TYPE_CHECKING: from noteflow.infrastructure.diarization import DiarizationEngine diff --git a/src/noteflow/grpc/server/_services.py b/src/noteflow/grpc/server/internal/services.py similarity index 90% rename from src/noteflow/grpc/server/_services.py rename to src/noteflow/grpc/server/internal/services.py index 8b2a75a..5bab4d3 100644 --- a/src/noteflow/grpc/server/_services.py +++ b/src/noteflow/grpc/server/internal/services.py @@ -5,21 +5,21 @@ from __future__ import annotations from typing import TYPE_CHECKING, Protocol from noteflow.infrastructure.logging import get_logger -from noteflow.application.services.streaming_config_persistence import StreamingConfig +from noteflow.application.services.streaming_config import StreamingConfig from noteflow.infrastructure.summarization import create_summarization_service -from .._config import ServicesConfig -from .._server_bootstrap import create_consent_persist_callback, load_consent_from_db -from ..service import NoteFlowServicer +from ...config.config import ServicesConfig +from ...startup.server_bootstrap import create_consent_persist_callback, load_consent_from_db +from ...service import NoteFlowServicer if TYPE_CHECKING: from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker from noteflow.application.services.calendar import CalendarService - from noteflow.application.services.ner_service import NerService + from noteflow.application.services.ner import NerService from noteflow.application.services.project_service import ProjectService from noteflow.application.services.summarization import SummarizationService - from noteflow.application.services.webhook_service import WebhookService + from noteflow.application.services.webhooks import WebhookService from noteflow.config.settings import Settings from noteflow.infrastructure.diarization import DiarizationEngine diff --git a/src/noteflow/grpc/server/_setup.py b/src/noteflow/grpc/server/internal/setup.py similarity index 86% rename from src/noteflow/grpc/server/_setup.py rename to src/noteflow/grpc/server/internal/setup.py index 03e05ba..14a26be 100644 --- a/src/noteflow/grpc/server/_setup.py +++ b/src/noteflow/grpc/server/internal/setup.py @@ -6,11 +6,11 @@ from typing import TYPE_CHECKING, cast import grpc.aio -from ..interceptors import IdentityInterceptor, RequestLoggingInterceptor -from ..proto import noteflow_pb2_grpc +from ...interceptors import IdentityInterceptor, RequestLoggingInterceptor +from ...proto import noteflow_pb2_grpc if TYPE_CHECKING: - from ..service import NoteFlowServicer + from ...service import NoteFlowServicer def create_server() -> grpc.aio.Server: @@ -44,6 +44,6 @@ def bind_server( bound_port = server.add_insecure_port(address) if bound_port == 0: raise RuntimeError(f"Failed to bind gRPC server on {address}") - if port == 0 and bound_port != 0: + if port == 0: address = f"{bind_address}:{bound_port}" return address diff --git a/src/noteflow/grpc/server/_streaming_config.py b/src/noteflow/grpc/server/internal/streaming_config.py similarity index 96% rename from src/noteflow/grpc/server/_streaming_config.py rename to src/noteflow/grpc/server/internal/streaming_config.py index 3826a5c..67a2b4d 100644 --- a/src/noteflow/grpc/server/_streaming_config.py +++ b/src/noteflow/grpc/server/internal/streaming_config.py @@ -5,7 +5,7 @@ from __future__ import annotations from pathlib import Path from typing import TYPE_CHECKING -from noteflow.application.services.streaming_config_persistence import ( +from noteflow.application.services.streaming_config import ( StreamingConfig, resolve_streaming_config_preference, ) diff --git a/src/noteflow/grpc/server/_types.py b/src/noteflow/grpc/server/internal/types.py similarity index 80% rename from src/noteflow/grpc/server/_types.py rename to src/noteflow/grpc/server/internal/types.py index 1d49bb7..36ef938 100644 --- a/src/noteflow/grpc/server/_types.py +++ b/src/noteflow/grpc/server/internal/types.py @@ -7,8 +7,8 @@ from typing import TYPE_CHECKING, TypedDict if TYPE_CHECKING: from sqlalchemy.ext.asyncio import AsyncEngine, AsyncSession, async_sessionmaker - from .._config import AsrConfig, ServicesConfig - from noteflow.application.services.streaming_config_persistence import StreamingConfig + from ...config.config import AsrConfig, ServicesConfig + from noteflow.application.services.streaming_config import StreamingConfig class ServerInitKwargs(TypedDict, total=False): diff --git a/src/noteflow/grpc/service.py b/src/noteflow/grpc/service.py index 0137a24..e90da5a 100644 --- a/src/noteflow/grpc/service.py +++ b/src/noteflow/grpc/service.py @@ -11,10 +11,10 @@ from typing import TYPE_CHECKING, ClassVar, Final from noteflow import __version__ from noteflow.config.constants import APP_DIR_NAME, SETTING_ASR_CONFIG from noteflow.config.settings import get_settings -from noteflow.application.services.asr_config_persistence import ( +from noteflow.application.services.asr_config import ( build_asr_config_preference, ) -from noteflow.application.services.streaming_config_persistence import ( +from noteflow.application.services.streaming_config import ( build_default_streaming_config, ) from noteflow.config.constants import DEFAULT_SAMPLE_RATE as _DEFAULT_SAMPLE_RATE @@ -26,9 +26,9 @@ from noteflow.infrastructure.persistence.repositories import DiarizationJob from noteflow.infrastructure.security.crypto import AesGcmCryptoBox from noteflow.infrastructure.security.keystore import KeyringKeyStore -from ._config import ServicesConfig -from ._identity_singleton import default_identity_service -from ._mixins import ( +from .config.config import ServicesConfig +from .identity.singleton import default_identity_service +from .mixins import ( AnnotationMixin, AsrConfigMixin, CalendarMixin, @@ -52,8 +52,8 @@ from ._mixins import ( SyncMixin, WebhooksMixin, ) -from ._service_base import GrpcBaseServicer, NoteFlowServicerStubs -from ._service_mixins import ( +from .servicer.base import GrpcBaseServicer, NoteFlowServicerStubs +from .servicer.mixins import ( ServicerAudioMixin, ServicerContextMixin, ServicerInfoMixin, @@ -66,9 +66,9 @@ from .stream_state import MeetingStreamState if TYPE_CHECKING: from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker - from noteflow.application.services.asr_config_types import AsrCapabilities - from noteflow.application.services.asr_config_service import AsrConfigService - from noteflow.application.services.hf_token_service import HfTokenService + from noteflow.application.services.asr_config import AsrCapabilities + from noteflow.application.services.asr_config import AsrConfigService + from noteflow.application.services.huggingface import HfTokenService from noteflow.infrastructure.asr import FasterWhisperEngine from noteflow.infrastructure.auth.oidc_registry import OidcAuthService @@ -113,7 +113,7 @@ class NoteFlowServicer( use `self: Protocol` annotations. """ - # Type stubs now live in _service_stubs.py to keep module size down. + # Type stubs now live in servicer/stubs.py to keep module size down. VERSION: Final[str] = __version__ MAX_CHUNK_SIZE: Final[int] = 1024 * 1024 # 1MB @@ -176,10 +176,10 @@ class NoteFlowServicer( session_factory: async_sessionmaker[AsyncSession] | None, ) -> None: """Initialize audio recording infrastructure.""" - from noteflow.application.services.asr_config_service import ( + from noteflow.application.services.asr_config import ( AsrConfigService as AsrConfigServiceImpl, ) - from noteflow.application.services.hf_token_service import ( + from noteflow.application.services.huggingface import ( HfTokenService as HfTokenServiceImpl, ) diff --git a/src/noteflow/grpc/servicer/__init__.py b/src/noteflow/grpc/servicer/__init__.py new file mode 100644 index 0000000..b1fa5fe --- /dev/null +++ b/src/noteflow/grpc/servicer/__init__.py @@ -0,0 +1 @@ +"""Service scaffolding for gRPC servicer.""" diff --git a/src/noteflow/grpc/_service_base.py b/src/noteflow/grpc/servicer/base.py similarity index 92% rename from src/noteflow/grpc/_service_base.py rename to src/noteflow/grpc/servicer/base.py index e446ce2..60c56b4 100644 --- a/src/noteflow/grpc/_service_base.py +++ b/src/noteflow/grpc/servicer/base.py @@ -4,7 +4,7 @@ from __future__ import annotations from typing import TYPE_CHECKING -from .proto import noteflow_pb2_grpc +from ..proto import noteflow_pb2_grpc if TYPE_CHECKING: GrpcBaseServicer = object diff --git a/src/noteflow/grpc/_service_mixins.py b/src/noteflow/grpc/servicer/mixins.py similarity index 98% rename from src/noteflow/grpc/_service_mixins.py rename to src/noteflow/grpc/servicer/mixins.py index 409fb1f..44b985f 100644 --- a/src/noteflow/grpc/_service_mixins.py +++ b/src/noteflow/grpc/servicer/mixins.py @@ -13,7 +13,7 @@ from noteflow.domain.identity.context import OperationContext, UserContext, Work from noteflow.domain.identity.roles import WorkspaceRole from noteflow.domain.value_objects import MeetingState from noteflow.grpc.meeting_store import MeetingStore -from noteflow.application.services.streaming_config_persistence import StreamingConfig +from noteflow.application.services.streaming_config import StreamingConfig from noteflow.infrastructure.asr import Segmenter, SegmenterConfig, StreamingVad from noteflow.infrastructure.audio.partial_buffer import PartialAudioBuffer from noteflow.infrastructure.audio.writer import MeetingAudioWriter @@ -28,7 +28,7 @@ from noteflow.infrastructure.persistence.memory import MemoryUnitOfWork from noteflow.infrastructure.persistence.unit_of_work import SqlAlchemyUnitOfWork from noteflow.infrastructure.security.crypto import AesGcmCryptoBox -from ._service_shutdown import ( +from .shutdown import ( cancel_diarization_tasks, cancel_sync_tasks, close_audio_writers, @@ -37,8 +37,8 @@ from ._service_shutdown import ( mark_in_memory_jobs_failed, mark_running_jobs_failed_db, ) -from .proto import noteflow_pb2 -from .stream_state import MeetingStreamState +from ..proto import noteflow_pb2 +from ..stream_state import MeetingStreamState if TYPE_CHECKING: from collections.abc import Awaitable, Callable @@ -48,7 +48,7 @@ if TYPE_CHECKING: from noteflow.infrastructure.asr import FasterWhisperEngine from noteflow.infrastructure.diarization.engine import DiarizationEngine -from ._mixins._types import GrpcContext +from ..mixins._types import GrpcContext logger = get_logger(__name__) class ServicerUowMixin: diff --git a/src/noteflow/grpc/_service_shutdown.py b/src/noteflow/grpc/servicer/shutdown.py similarity index 98% rename from src/noteflow/grpc/_service_shutdown.py rename to src/noteflow/grpc/servicer/shutdown.py index 2c1ec94..1403eb6 100644 --- a/src/noteflow/grpc/_service_shutdown.py +++ b/src/noteflow/grpc/servicer/shutdown.py @@ -9,7 +9,7 @@ from typing import TYPE_CHECKING from noteflow.infrastructure.logging import get_logger if TYPE_CHECKING: - from .service import NoteFlowServicer + from ..service import NoteFlowServicer logger = get_logger(__name__) @@ -56,7 +56,7 @@ def mark_in_memory_jobs_failed( cancelled_job_ids: list[str], ) -> None: """Mark in-memory diarization jobs as failed.""" - from .proto import noteflow_pb2 + from ..proto import noteflow_pb2 if servicer.session_factory is not None or not cancelled_job_ids: return diff --git a/src/noteflow/grpc/_service_stubs.py b/src/noteflow/grpc/servicer/stubs.py similarity index 98% rename from src/noteflow/grpc/_service_stubs.py rename to src/noteflow/grpc/servicer/stubs.py index 7efa45e..14c0cc0 100644 --- a/src/noteflow/grpc/_service_stubs.py +++ b/src/noteflow/grpc/servicer/stubs.py @@ -5,8 +5,8 @@ from __future__ import annotations from collections.abc import AsyncIterator from typing import Protocol -from ._mixins._types import GrpcContext, GrpcStatusContext -from .proto import noteflow_pb2 +from ..mixins._types import GrpcContext, GrpcStatusContext +from ..proto import noteflow_pb2 class _StreamingStubs(Protocol): diff --git a/src/noteflow/grpc/startup/__init__.py b/src/noteflow/grpc/startup/__init__.py new file mode 100644 index 0000000..8f0fa9b --- /dev/null +++ b/src/noteflow/grpc/startup/__init__.py @@ -0,0 +1 @@ +"""Startup helpers for gRPC server.""" diff --git a/src/noteflow/grpc/_startup_banner.py b/src/noteflow/grpc/startup/banner.py similarity index 96% rename from src/noteflow/grpc/_startup_banner.py rename to src/noteflow/grpc/startup/banner.py index 95dd4f8..a5573ee 100644 --- a/src/noteflow/grpc/_startup_banner.py +++ b/src/noteflow/grpc/startup/banner.py @@ -11,9 +11,9 @@ from noteflow.config.constants import STATUS_DISABLED from noteflow.infrastructure.logging import get_logger if TYPE_CHECKING: - from noteflow.application.services.webhook_service import WebhookService + from noteflow.application.services.webhooks import WebhookService - from ._startup import GrpcServerConfigLike, StartupServices + from .startup import GrpcServerConfigLike, StartupServices logger = get_logger(__name__) diff --git a/src/noteflow/grpc/_server_bootstrap.py b/src/noteflow/grpc/startup/server_bootstrap.py similarity index 100% rename from src/noteflow/grpc/_server_bootstrap.py rename to src/noteflow/grpc/startup/server_bootstrap.py diff --git a/src/noteflow/grpc/_startup_services.py b/src/noteflow/grpc/startup/services.py similarity index 97% rename from src/noteflow/grpc/_startup_services.py rename to src/noteflow/grpc/startup/services.py index 12d8cae..5dd8334 100644 --- a/src/noteflow/grpc/_startup_services.py +++ b/src/noteflow/grpc/startup/services.py @@ -7,8 +7,8 @@ from typing import TYPE_CHECKING, TypedDict from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker from noteflow.application.services.calendar import CalendarService -from noteflow.application.services.ner_service import NerService -from noteflow.application.services.webhook_service import WebhookService +from noteflow.application.services.ner import NerService +from noteflow.application.services.webhooks import WebhookService from noteflow.config.settings import Settings, get_calendar_settings, get_feature_flags from noteflow.domain.constants.fields import CALENDAR from noteflow.domain.entities.integration import IntegrationStatus @@ -20,7 +20,7 @@ from noteflow.infrastructure.persistence.unit_of_work import SqlAlchemyUnitOfWor from noteflow.infrastructure.webhooks import WebhookExecutor if TYPE_CHECKING: - from ._startup import DiarizationConfigLike + from .startup import DiarizationConfigLike logger = get_logger(__name__) diff --git a/src/noteflow/grpc/_startup.py b/src/noteflow/grpc/startup/startup.py similarity index 98% rename from src/noteflow/grpc/_startup.py rename to src/noteflow/grpc/startup/startup.py index f4f85ea..8b856bf 100644 --- a/src/noteflow/grpc/_startup.py +++ b/src/noteflow/grpc/startup/startup.py @@ -32,8 +32,8 @@ from noteflow.infrastructure.persistence.unit_of_work import SqlAlchemyUnitOfWor from noteflow.infrastructure.summarization import CloudBackend, CloudSummarizer # Re-export for backward compatibility -from ._startup_banner import print_startup_banner -from ._startup_services import ( +from .banner import print_startup_banner +from .services import ( check_calendar_needed_from_db, create_calendar_service, create_diarization_engine, @@ -238,7 +238,7 @@ async def setup_summarization_with_consent( if TYPE_CHECKING: from noteflow.application.services.calendar import CalendarService - from noteflow.application.services.webhook_service import WebhookService + from noteflow.application.services.webhooks import WebhookService from noteflow.infrastructure.diarization import DiarizationEngine diff --git a/src/noteflow/grpc/types/__init__.py b/src/noteflow/grpc/types/__init__.py new file mode 100644 index 0000000..c826f07 --- /dev/null +++ b/src/noteflow/grpc/types/__init__.py @@ -0,0 +1,203 @@ +"""Data types for NoteFlow gRPC client operations.""" + +from __future__ import annotations + +from collections.abc import Callable +from dataclasses import dataclass +from enum import StrEnum +from typing import Generic, TypeVar, Never + +import grpc + +T = TypeVar("T", covariant=True) +U = TypeVar("U") +E = TypeVar("E", bound=BaseException) + + +class ClientErrorCode(StrEnum): + """Client-facing error codes for gRPC operations.""" + + UNKNOWN = "unknown" + NOT_CONNECTED = "not_connected" + NOT_FOUND = "not_found" + INVALID_ARGUMENT = "invalid_argument" + PERMISSION_DENIED = "permission_denied" + DEADLINE_EXCEEDED = "deadline_exceeded" + ALREADY_EXISTS = "already_exists" + FAILED_PRECONDITION = "failed_precondition" + UNIMPLEMENTED = "unimplemented" + INTERNAL = "internal" + UNAVAILABLE = "unavailable" + + +@dataclass(frozen=True, slots=True) +class ClientError: + """Client error details for failed gRPC operations.""" + + code: ClientErrorCode + message: str + grpc_status: grpc.StatusCode | None = None + + @property + def is_retryable(self) -> bool: + """Return True for transient errors.""" + return self.code in { + ClientErrorCode.UNAVAILABLE, + ClientErrorCode.DEADLINE_EXCEEDED, + ClientErrorCode.INTERNAL, + } + + @property + def is_not_found(self) -> bool: + """Return True for NOT_FOUND errors.""" + return self.code == ClientErrorCode.NOT_FOUND + + +@dataclass(frozen=True, slots=True) +class ClientResult(Generic[T]): + """Result wrapper for client operations.""" + + value: T | None + error: ClientError | None + + @property + def success(self) -> bool: + """Return True if the operation succeeded.""" + return self.error is None + + @property + def failed(self) -> bool: + """Return True if the operation failed.""" + return self.error is not None + + def unwrap(self) -> T: + """Return value or raise on failure.""" + if self.error is not None: + raise RuntimeError(f"Client operation failed: {self.error.message}") + if self.value is None: + raise RuntimeError("Client operation returned no value") + return self.value + + def unwrap_or(self, default: U) -> T | U: + """Return value or default on failure.""" + return self.value if self.error is None and self.value is not None else default + + +def Ok(value: T) -> ClientResult[T]: + """Create a successful ClientResult.""" + return ClientResult(value=value, error=None) + + +def Err( + *, + code: ClientErrorCode, + message: str, + grpc_status: grpc.StatusCode | None = None, +) -> ClientResult[Never]: + """Create a failed ClientResult.""" + return ClientResult( + value=None, + error=ClientError(code=code, message=message, grpc_status=grpc_status), + ) + + +@dataclass +class TranscriptSegment: + """Transcript segment from server.""" + + segment_id: int + text: str + start_time: float + end_time: float + language: str + is_final: bool + speaker_id: str = "" + speaker_confidence: float = 0.0 + + +@dataclass +class ServerInfo: + """Server information.""" + + version: str + asr_model: str + asr_ready: bool + uptime_seconds: float + active_meetings: int + diarization_enabled: bool = False + diarization_ready: bool = False + system_ram_total_bytes: int | None = None + system_ram_available_bytes: int | None = None + gpu_vram_total_bytes: int | None = None + gpu_vram_available_bytes: int | None = None + + +@dataclass +class MeetingInfo: + """Meeting information.""" + + id: str + title: str + state: str + created_at: float + started_at: float + ended_at: float + duration_seconds: float + segment_count: int + + +@dataclass +class AnnotationInfo: + """Annotation information.""" + + id: str + meeting_id: str + annotation_type: str + text: str + start_time: float + end_time: float + segment_ids: list[int] + created_at: float + + +@dataclass +class ExportResult: + """Export result.""" + + content: str + format_name: str + file_extension: str + + +@dataclass +class DiarizationResult: + """Result of speaker diarization refinement.""" + + job_id: str + status: str + segments_updated: int + speaker_ids: list[str] + error_message: str = "" + + @property + def success(self) -> bool: + """Check if diarization succeeded.""" + return self.status == "completed" and not self.error_message + + @property + def is_terminal(self) -> bool: + """Check if job reached a terminal state.""" + return self.status in {"completed", "failed"} + + +@dataclass +class RenameSpeakerResult: + """Result of speaker rename operation.""" + + segments_updated: int + success: bool + + +# Callback types +TranscriptCallback = Callable[[TranscriptSegment], None] +ConnectionCallback = Callable[[bool, str], None] diff --git a/src/noteflow/infrastructure/audio/partial_buffer.py b/src/noteflow/infrastructure/audio/partial_buffer.py index 7a08497..50ada00 100644 --- a/src/noteflow/infrastructure/audio/partial_buffer.py +++ b/src/noteflow/infrastructure/audio/partial_buffer.py @@ -91,6 +91,9 @@ class PartialAudioBuffer: # Buffer overflow - this indicates a bug in the caller # (should have read and cleared within the partial cadence) # Drop the samples rather than corrupt memory + from noteflow.infrastructure.metrics import get_infrastructure_metrics + + get_infrastructure_metrics().record_buffer_overflow("partial_audio", samples) return False # Copy samples into pre-allocated buffer (no allocation) diff --git a/src/noteflow/infrastructure/auth/_presets.py b/src/noteflow/infrastructure/auth/_presets.py index 7a1ab27..d95267f 100644 --- a/src/noteflow/infrastructure/auth/_presets.py +++ b/src/noteflow/infrastructure/auth/_presets.py @@ -12,6 +12,7 @@ from noteflow.domain.auth.oidc import ( ClaimMapping, OidcProviderPreset, ) +from noteflow.domain.constants.fields import DISPLAY_NAME from noteflow.domain.auth.oidc_constants import ( CLAIM_EMAIL, CLAIM_EMAIL_VERIFIED, @@ -46,7 +47,7 @@ class ProviderPresetConfig: """Convert preset config to dictionary representation.""" return { FIELD_PRESET: self.preset.value, - "display_name": self.display_name, + DISPLAY_NAME: self.display_name, "description": self.description, "default_scopes": list(self.default_scopes), "documentation_url": self.documentation_url, diff --git a/src/noteflow/infrastructure/auth/oidc_registry.py b/src/noteflow/infrastructure/auth/oidc_registry.py index f0ef896..bec7cfb 100644 --- a/src/noteflow/infrastructure/auth/oidc_registry.py +++ b/src/noteflow/infrastructure/auth/oidc_registry.py @@ -320,7 +320,11 @@ class OidcAuthService: """ all_presets = get_all_preset_options() # Filter out CUSTOM template and sort by name for consistent ordering - filtered = [p for p in all_presets if p.get("preset") != "custom"] + filtered = [ + p + for p in all_presets + if p.get("preset") != OidcProviderPreset.CUSTOM.value + ] filtered.sort(key=lambda p: str(p.get("name", ""))) logger.debug("Returning %d OIDC provider presets", len(filtered)) return filtered diff --git a/src/noteflow/infrastructure/calendar/google_adapter.py b/src/noteflow/infrastructure/calendar/google_adapter.py index bcdf59d..69d4d05 100644 --- a/src/noteflow/infrastructure/calendar/google_adapter.py +++ b/src/noteflow/infrastructure/calendar/google_adapter.py @@ -115,7 +115,8 @@ class GoogleCalendarAdapter(CalendarPort): hours_ahead=hours_ahead, ) - return [self._parse_google_event(item) for item in items] + parsed = [self._parse_google_event(item) for item in items] + return [event for event in parsed if event is not None] def _build_events_request( @@ -205,9 +206,7 @@ class GoogleCalendarAdapter(CalendarPort): if not email: raise GoogleCalendarError("No email in userinfo response") - # Get display name from 'name' field, fall back to email prefix - name = data.get("name") - if name: + if name := data.get("name"): display_name = str(name) else: # Extract username from email, handling edge cases where @ may be missing @@ -220,35 +219,28 @@ class GoogleCalendarAdapter(CalendarPort): get_user_email = user_email get_user_info = user_info - def _parse_google_event(self, item: _GoogleEvent) -> CalendarEventInfo: - """Parse Google Calendar event into CalendarEventInfo.""" + def _parse_google_event(self, item: _GoogleEvent) -> CalendarEventInfo | None: + """Parse Google Calendar event into CalendarEventInfo. + + Returns: + CalendarEventInfo or None if event has invalid datetime data. + """ event_id = str(item.get("id", "")) title = str(item.get("summary", DEFAULT_MEETING_TITLE)) - - # Parse start/end times - start_data = item.get(START, {}) - end_data = item.get("end", {}) - + start_data, end_data = item.get(START, {}), item.get("end", {}) is_all_day = DATE in start_data - start_time = self._parse_google_datetime(start_data) - end_time = self._parse_google_datetime(end_data) + start_time, end_time = self._parse_google_datetime(start_data), self._parse_google_datetime(end_data) - # Parse attendees - attendees_data = item.get(ATTENDEES, []) - attendees = tuple( - str(attendee.get(EMAIL, "")) - for attendee in attendees_data - if attendee.get(EMAIL) - ) + if start_time is None or end_time is None: + logger.warning( + "google_event_skipped_invalid_datetime", + event_id=event_id, title=title, + has_start=start_time is not None, has_end=end_time is not None, + ) + return None - # Extract meeting URL from conferenceData or hangoutLink - meeting_url = self._extract_meeting_url(item) - - # Check if recurring - is_recurring = bool(item.get("recurringEventId")) - - location = item.get(LOCATION) - description = item.get("description") + attendees = self._parse_google_attendees(item.get(ATTENDEES, [])) + location, description = item.get(LOCATION), item.get("description") return CalendarEventInfo( id=event_id, @@ -258,20 +250,28 @@ class GoogleCalendarAdapter(CalendarPort): attendees=attendees, location=str(location) if location else None, description=str(description) if description else None, - meeting_url=meeting_url, - is_recurring=is_recurring, + meeting_url=self._extract_meeting_url(item), + is_recurring=bool(item.get("recurringEventId")), is_all_day=is_all_day, provider=OAuthProvider.GOOGLE, raw=dict(item), ) - def _parse_google_datetime(self, dt_data: _GoogleEventDateTime) -> datetime: - """Parse datetime from Google Calendar format.""" + def _parse_google_datetime(self, dt_data: _GoogleEventDateTime) -> datetime | None: + """Parse datetime from Google Calendar format. + + Returns: + Parsed datetime or None if parsing fails. + """ # All-day events use "date", timed events use "dateTime" dt_str = dt_data.get("dateTime") or dt_data.get(DATE) if not dt_str: - return datetime.now(UTC) + logger.warning( + "google_datetime_missing", + data_keys=list(dt_data.keys()), + ) + return None # Handle Z suffix for UTC if dt_str.endswith("Z"): @@ -284,8 +284,20 @@ class GoogleCalendarAdapter(CalendarPort): dt = dt.replace(tzinfo=UTC) return dt except ValueError: - logger.warning("Failed to parse datetime: %s", dt_str) - return datetime.now(UTC) + logger.warning( + "google_datetime_parse_failed", + original_value=dt_str, + ) + return None + + @staticmethod + def _parse_google_attendees(attendees_data: list[_GoogleAttendee]) -> tuple[str, ...]: + """Extract attendee emails from Google Calendar format.""" + return tuple( + str(attendee.get(EMAIL, "")) + for attendee in attendees_data + if attendee.get(EMAIL) + ) def _extract_meeting_url(self, item: _GoogleEvent) -> str | None: """Extract video meeting URL from event data.""" diff --git a/src/noteflow/infrastructure/calendar/oauth_flow.py b/src/noteflow/infrastructure/calendar/oauth_flow.py index 6e76f62..aabf023 100644 --- a/src/noteflow/infrastructure/calendar/oauth_flow.py +++ b/src/noteflow/infrastructure/calendar/oauth_flow.py @@ -20,7 +20,7 @@ from noteflow.config.constants import ( PKCE_CODE_VERIFIER_BYTES, ) from noteflow.config.constants.encoding import ASCII_ENCODING -from noteflow.domain.constants.fields import CODE +from noteflow.domain.constants.fields import CODE, STATE from noteflow.domain.value_objects import OAuthProvider, OAuthState, OAuthTokens @@ -127,7 +127,7 @@ def build_auth_url(config: AuthUrlConfig) -> str: "redirect_uri": config.redirect_uri, "response_type": CODE, OAUTH_FIELD_SCOPE: " ".join(config.scopes), - "state": config.state, + STATE: config.state, "code_challenge": config.code_challenge, "code_challenge_method": "S256", } diff --git a/src/noteflow/infrastructure/calendar/outlook/_event_fetcher.py b/src/noteflow/infrastructure/calendar/outlook/_event_fetcher.py index 31d01da..d21bcf7 100644 --- a/src/noteflow/infrastructure/calendar/outlook/_event_fetcher.py +++ b/src/noteflow/infrastructure/calendar/outlook/_event_fetcher.py @@ -85,7 +85,7 @@ def accumulate_events( items: list[OutlookEvent], all_events: list[CalendarEventInfo], limit: int, - parse_event_func: Callable[[OutlookEvent], CalendarEventInfo], + parse_event_func: Callable[[OutlookEvent], CalendarEventInfo | None], ) -> tuple[list[CalendarEventInfo], bool]: """Accumulate parsed events up to the limit. @@ -94,12 +94,16 @@ def accumulate_events( all_events: Accumulated events so far. limit: Maximum events to collect. parse_event_func: Function to parse OutlookEvent to CalendarEventInfo. + Returns None for events with invalid datetime data. Returns: Tuple of (updated events list, whether limit was reached). """ for item in items: - all_events.append(parse_event_func(item)) + parsed = parse_event_func(item) + if parsed is None: + continue + all_events.append(parsed) if len(all_events) >= limit: return all_events, True return all_events, False diff --git a/src/noteflow/infrastructure/calendar/outlook/_event_parser.py b/src/noteflow/infrastructure/calendar/outlook/_event_parser.py index 3e626d6..e120595 100644 --- a/src/noteflow/infrastructure/calendar/outlook/_event_parser.py +++ b/src/noteflow/infrastructure/calendar/outlook/_event_parser.py @@ -18,37 +18,27 @@ from noteflow.infrastructure.logging import get_logger logger = get_logger(__name__) -def parse_outlook_event(item: OutlookEvent) -> CalendarEventInfo: - """Parse Microsoft Graph event into CalendarEventInfo.""" +def parse_outlook_event(item: OutlookEvent) -> CalendarEventInfo | None: + """Parse Microsoft Graph event into CalendarEventInfo. + + Returns: + CalendarEventInfo or None if event has invalid datetime data. + """ event_id = str(item.get("id", "")) title = str(item.get("subject", DEFAULT_MEETING_TITLE)) + start_data, end_data = item.get(START, {}), item.get("end", {}) + start_time, end_time = parse_outlook_datetime(start_data), parse_outlook_datetime(end_data) - # Parse start/end times - start_data = item.get(START, {}) - end_data = item.get("end", {}) + if start_time is None or end_time is None: + logger.warning( + "outlook_event_skipped_invalid_datetime", + event_id=event_id, title=title, + has_start=start_time is not None, has_end=end_time is not None, + ) + return None - start_time = parse_outlook_datetime(start_data) - end_time = parse_outlook_datetime(end_data) - - # Check if all-day event - is_all_day = bool(item.get("isAllDay", False)) - - # Parse attendees - attendees_data = item.get(ATTENDEES, []) - attendees = parse_attendees(attendees_data) - - # Extract meeting URL - meeting_url = extract_meeting_url(item) - - # Check if recurring (has seriesMasterId) - is_recurring = bool(item.get("seriesMasterId")) - - # Location location_data = item.get(LOCATION, {}) raw_location = location_data.get("displayName") - location = str(raw_location) if raw_location else None - - # Description (bodyPreview) description = item.get("bodyPreview") return CalendarEventInfo( @@ -56,24 +46,32 @@ def parse_outlook_event(item: OutlookEvent) -> CalendarEventInfo: title=title, start_time=start_time, end_time=end_time, - attendees=attendees, - location=location, + attendees=parse_attendees(item.get(ATTENDEES, [])), + location=str(raw_location) if raw_location else None, description=str(description) if description else None, - meeting_url=meeting_url, - is_recurring=is_recurring, - is_all_day=is_all_day, + meeting_url=extract_meeting_url(item), + is_recurring=bool(item.get("seriesMasterId")), + is_all_day=bool(item.get("isAllDay", False)), provider=OAuthProvider.OUTLOOK, raw=dict(item), ) -def parse_outlook_datetime(dt_data: OutlookDateTime) -> datetime: - """Parse datetime from Microsoft Graph format.""" +def parse_outlook_datetime(dt_data: OutlookDateTime) -> datetime | None: + """Parse datetime from Microsoft Graph format. + + Returns: + Parsed datetime or None if parsing fails. + """ dt_str = dt_data.get("dateTime") timezone = dt_data.get("timeZone", "UTC") if not dt_str: - return datetime.now(UTC) + logger.warning( + "outlook_datetime_missing", + data_keys=list(dt_data.keys()), + ) + return None try: # Graph API returns ISO format without timezone suffix @@ -83,8 +81,12 @@ def parse_outlook_datetime(dt_data: OutlookDateTime) -> datetime: dt = dt.replace(tzinfo=UTC) return dt except ValueError: - logger.warning("Failed to parse datetime: %s (tz: %s)", dt_str, timezone) - return datetime.now(UTC) + logger.warning( + "outlook_datetime_parse_failed", + original_value=dt_str, + timezone=timezone, + ) + return None def parse_attendees(attendees_data: list[OutlookAttendee]) -> tuple[str, ...]: diff --git a/src/noteflow/infrastructure/converters/integration_converters.py b/src/noteflow/infrastructure/converters/integration_converters.py index 05126ca..ef53616 100644 --- a/src/noteflow/infrastructure/converters/integration_converters.py +++ b/src/noteflow/infrastructure/converters/integration_converters.py @@ -4,7 +4,7 @@ from __future__ import annotations from typing import TYPE_CHECKING -from noteflow.domain.constants.fields import DURATION_MS +from noteflow.domain.constants.fields import DURATION_MS, ENDED_AT from noteflow.domain.entities.integration import ( Integration, IntegrationStatus, @@ -115,7 +115,7 @@ class SyncRunConverter: "integration_id": entity.integration_id, "status": entity.status.value, "started_at": entity.started_at, - "ended_at": entity.ended_at, + ENDED_AT: entity.ended_at, DURATION_MS: entity.duration_ms, "error_message": entity.error_message, "stats": entity.stats, diff --git a/src/noteflow/infrastructure/metrics/__init__.py b/src/noteflow/infrastructure/metrics/__init__.py index 6e21b6b..9d68af2 100644 --- a/src/noteflow/infrastructure/metrics/__init__.py +++ b/src/noteflow/infrastructure/metrics/__init__.py @@ -1,12 +1,22 @@ """Metrics infrastructure for NoteFlow.""" from .collector import MetricsCollector, PerformanceMetrics, get_metrics_collector +from .infrastructure_metrics import ( + InfrastructureMetrics, + InfrastructureStats, + get_infrastructure_metrics, + reset_infrastructure_metrics, +) from .system_resources import SystemResources, get_system_resources __all__ = [ "MetricsCollector", "PerformanceMetrics", "get_metrics_collector", + "InfrastructureMetrics", + "InfrastructureStats", + "get_infrastructure_metrics", + "reset_infrastructure_metrics", "SystemResources", "get_system_resources", ] diff --git a/src/noteflow/infrastructure/metrics/infrastructure_metrics.py b/src/noteflow/infrastructure/metrics/infrastructure_metrics.py new file mode 100644 index 0000000..7d4742b --- /dev/null +++ b/src/noteflow/infrastructure/metrics/infrastructure_metrics.py @@ -0,0 +1,257 @@ +"""Infrastructure metrics tracking. + +Provides visibility into infrastructure fallbacks, buffer overflows, and provider availability. + +Sprint GAP-003: Error Handling Mismatches - Swallowed Exception Observability +""" + +from __future__ import annotations + +import time +from collections import defaultdict +from dataclasses import dataclass, field +from threading import Lock +from typing import Final + +from noteflow.infrastructure.logging import get_logger + +_logger = get_logger(__name__) + +# Metric window for rolling stats (5 minutes) +METRIC_WINDOW_SECONDS: Final[int] = 3 * 100 + + +@dataclass(frozen=True, slots=True) +class InfrastructureStats: + """Point-in-time infrastructure statistics.""" + + total_fallbacks: int + fallbacks_by_component: dict[str, int] + total_buffer_overflows: int + samples_dropped: int + unavailable_providers: set[str] + + @classmethod + def empty(cls) -> InfrastructureStats: + """Create empty stats representing no infrastructure events. + + Returns a stats instance with zero counts, representing a pristine state + with no fallbacks, overflows, or provider issues recorded. + + Returns: + InfrastructureStats with all counts zeroed. + """ + _logger.debug("Creating empty InfrastructureStats (no events)") + return cls( + total_fallbacks=0, + fallbacks_by_component={}, + total_buffer_overflows=0, + samples_dropped=0, + unavailable_providers=set(), + ) + + +@dataclass +class _FallbackMetric: + """Single configuration fallback metric.""" + + timestamp: float + component: str + error_type: str + + +@dataclass +class _BufferOverflowMetric: + """Single buffer overflow metric.""" + + timestamp: float + buffer_name: str + samples_dropped: int + + +@dataclass +class _ProviderUnavailableMetric: + """Single provider unavailable metric.""" + + timestamp: float + provider: str + reason: str + + +@dataclass +class _MetricsBuffer: + """Thread-safe buffer for rolling metrics.""" + + fallbacks: list[_FallbackMetric] = field(default_factory=list) + overflows: list[_BufferOverflowMetric] = field(default_factory=list) + provider_issues: list[_ProviderUnavailableMetric] = field(default_factory=list) + lock: Lock = field(default_factory=Lock) + + def append_fallback(self, metric: _FallbackMetric) -> None: + """Append a fallback metric and prune old entries.""" + cutoff = time.time() - METRIC_WINDOW_SECONDS + with self.lock: + self.fallbacks = [m for m in self.fallbacks if m.timestamp > cutoff] + self.fallbacks.append(metric) + + def append_overflow(self, metric: _BufferOverflowMetric) -> None: + """Append an overflow metric and prune old entries.""" + cutoff = time.time() - METRIC_WINDOW_SECONDS + with self.lock: + self.overflows = [m for m in self.overflows if m.timestamp > cutoff] + self.overflows.append(metric) + + def append_provider_issue(self, metric: _ProviderUnavailableMetric) -> None: + """Append a provider issue metric and prune old entries.""" + cutoff = time.time() - METRIC_WINDOW_SECONDS + with self.lock: + self.provider_issues = [m for m in self.provider_issues if m.timestamp > cutoff] + self.provider_issues.append(metric) + + def recent_after(self, cutoff: float) -> tuple[ + list[_FallbackMetric], + list[_BufferOverflowMetric], + list[_ProviderUnavailableMetric], + ]: + """Get recent metrics after cutoff time.""" + with self.lock: + return ( + [m for m in self.fallbacks if m.timestamp > cutoff], + [m for m in self.overflows if m.timestamp > cutoff], + [m for m in self.provider_issues if m.timestamp > cutoff], + ) + + +def _compute_stats( + fallbacks: list[_FallbackMetric], + overflows: list[_BufferOverflowMetric], + provider_issues: list[_ProviderUnavailableMetric], +) -> InfrastructureStats: + """Calculate current stats from metrics lists.""" + if not fallbacks and not overflows and not provider_issues: + return InfrastructureStats.empty() + + # Fallback stats + total_fallbacks = len(fallbacks) + fallbacks_by_component: dict[str, int] = defaultdict(int) + for metric in fallbacks: + fallbacks_by_component[metric.component] += 1 + + # Overflow stats + total_buffer_overflows = len(overflows) + samples_dropped = sum(m.samples_dropped for m in overflows) + + # Provider issues + unavailable_providers = {m.provider for m in provider_issues} + + return InfrastructureStats( + total_fallbacks=total_fallbacks, + fallbacks_by_component=dict(fallbacks_by_component), + total_buffer_overflows=total_buffer_overflows, + samples_dropped=samples_dropped, + unavailable_providers=unavailable_providers, + ) + + +class InfrastructureMetrics: + """Track infrastructure metrics. + + Provides rolling window statistics for infrastructure health monitoring. + Thread-safe for concurrent tracking. + """ + + def __init__(self) -> None: + """Initialize metrics tracker.""" + self._buffer = _MetricsBuffer() + + def record_fallback(self, component: str, error_type: str) -> None: + """Record a configuration fallback. + + Args: + component: Name of the component that fell back. + error_type: Type of error that triggered the fallback. + """ + metric = _FallbackMetric( + timestamp=time.time(), + component=component, + error_type=error_type, + ) + self._buffer.append_fallback(metric) + + _logger.info( + "infrastructure_fallback_recorded", + component=component, + error_type=error_type, + ) + + def record_buffer_overflow(self, buffer_name: str, samples_dropped: int) -> None: + """Record a buffer overflow event. + + Args: + buffer_name: Name of the buffer that overflowed. + samples_dropped: Number of audio samples dropped. + """ + metric = _BufferOverflowMetric( + timestamp=time.time(), + buffer_name=buffer_name, + samples_dropped=samples_dropped, + ) + self._buffer.append_overflow(metric) + + _logger.warning( + "infrastructure_buffer_overflow", + buffer_name=buffer_name, + samples_dropped=samples_dropped, + ) + + def record_provider_unavailable(self, provider: str, reason: str) -> None: + """Record a provider becoming unavailable. + + Args: + provider: Name of the provider. + reason: Reason for unavailability. + """ + metric = _ProviderUnavailableMetric( + timestamp=time.time(), + provider=provider, + reason=reason, + ) + self._buffer.append_provider_issue(metric) + + _logger.warning( + "infrastructure_provider_unavailable", + provider=provider, + reason=reason, + ) + + def get_infrastructure_stats(self) -> InfrastructureStats: + """Get current aggregate infrastructure statistics. + + Returns: + Rolling window statistics for all infrastructure events. + """ + cutoff = time.time() - METRIC_WINDOW_SECONDS + fallbacks, overflows, provider_issues = self._buffer.recent_after(cutoff) + return _compute_stats(fallbacks, overflows, provider_issues) + + +# Global singleton +_infrastructure_metrics: InfrastructureMetrics | None = None + + +def get_infrastructure_metrics() -> InfrastructureMetrics: + """Get the global infrastructure metrics singleton. + + Returns: + Shared InfrastructureMetrics instance. + """ + global _infrastructure_metrics + if _infrastructure_metrics is None: + _infrastructure_metrics = InfrastructureMetrics() + return _infrastructure_metrics + + +def reset_infrastructure_metrics() -> None: + """Reset the global infrastructure metrics singleton (testing helper).""" + global _infrastructure_metrics + _infrastructure_metrics = None diff --git a/src/noteflow/infrastructure/observability/usage/_database_sink.py b/src/noteflow/infrastructure/observability/usage/_database_sink.py index c6987c4..3c1a7db 100644 --- a/src/noteflow/infrastructure/observability/usage/_database_sink.py +++ b/src/noteflow/infrastructure/observability/usage/_database_sink.py @@ -213,9 +213,7 @@ class BufferedDatabaseUsageEventSink: Args: timeout: Maximum time to wait for flush to complete. """ - # Wait for any pending flush tasks - pending_tasks = [t for t in self._flush_tasks if not t.done()] - if pending_tasks: + if pending_tasks := [t for t in self._flush_tasks if not t.done()]: try: await asyncio.wait_for( asyncio.gather(*pending_tasks, return_exceptions=True), diff --git a/src/noteflow/infrastructure/persistence/models/_columns.py b/src/noteflow/infrastructure/persistence/models/_columns.py index 949eca9..946224a 100644 --- a/src/noteflow/infrastructure/persistence/models/_columns.py +++ b/src/noteflow/infrastructure/persistence/models/_columns.py @@ -48,8 +48,7 @@ def jsonb_dict_column() -> MappedColumn[dict[str, object]]: Returns: A mapped column configured for JSONB storage with ``{}`` default. """ - column = mapped_column(JSONB, nullable=False, default=dict) - return column + return mapped_column(JSONB, nullable=False, default=dict) def metadata_column() -> MappedColumn[dict[str, object]]: @@ -62,8 +61,7 @@ def metadata_column() -> MappedColumn[dict[str, object]]: Returns: A mapped column stored as ``metadata`` in PostgreSQL JSONB format. """ - column = mapped_column("metadata", JSONB, nullable=False, default=dict) - return column + return mapped_column("metadata", JSONB, nullable=False, default=dict) def utc_now_column() -> MappedColumn[datetime]: @@ -76,8 +74,7 @@ def utc_now_column() -> MappedColumn[datetime]: Returns: A mapped column with timezone-aware datetime, defaulting to ``utc_now()``. """ - column = mapped_column(DateTime(timezone=True), nullable=False, default=utc_now) - return column + return mapped_column(DateTime(timezone=True), nullable=False, default=utc_now) def utc_now_onupdate_column() -> MappedColumn[datetime]: @@ -90,13 +87,12 @@ def utc_now_onupdate_column() -> MappedColumn[datetime]: Returns: A mapped column with timezone-aware datetime, auto-updating on changes. """ - column = mapped_column( + return mapped_column( DateTime(timezone=True), nullable=False, default=utc_now, onupdate=utc_now, ) - return column def meeting_id_fk_column( @@ -119,7 +115,7 @@ def meeting_id_fk_column( Returns: A UUID foreign key column referencing ``noteflow.meetings.id``. """ - column = mapped_column( + return mapped_column( UUID(as_uuid=True), ForeignKey( FK_NOTEFLOW_MEETINGS_ID, ondelete=RELATIONSHIP_ON_DELETE_CASCADE @@ -128,7 +124,6 @@ def meeting_id_fk_column( index=index, unique=unique, ) - return column def workspace_id_fk_column( @@ -148,7 +143,7 @@ def workspace_id_fk_column( Returns: A UUID foreign key column referencing ``noteflow.workspaces.id``. """ - column = mapped_column( + return mapped_column( UUID(as_uuid=True), ForeignKey( "noteflow.workspaces.id", ondelete=RELATIONSHIP_ON_DELETE_CASCADE @@ -156,4 +151,3 @@ def workspace_id_fk_column( nullable=nullable, index=index, ) - return column diff --git a/src/noteflow/infrastructure/security/keystore.py b/src/noteflow/infrastructure/security/keystore.py index 9ba969c..b8f5d01 100644 --- a/src/noteflow/infrastructure/security/keystore.py +++ b/src/noteflow/infrastructure/security/keystore.py @@ -149,7 +149,11 @@ class KeyringKeyStore: try: stored = keyring.get_password(self._service_name, self._key_name) return stored is not None - except keyring.errors.KeyringError: + except keyring.errors.KeyringError as e: + logger.debug( + "keyring_existence_check_failed", + error_type=type(e).__name__, + ) return False get_or_create_master_key = master_key diff --git a/src/noteflow/infrastructure/summarization/ollama_provider.py b/src/noteflow/infrastructure/summarization/ollama_provider.py index a36b37e..2a348c9 100644 --- a/src/noteflow/infrastructure/summarization/ollama_provider.py +++ b/src/noteflow/infrastructure/summarization/ollama_provider.py @@ -11,6 +11,7 @@ from typing import TYPE_CHECKING, Protocol, cast from noteflow.config.constants.core import ( DEFAULT_LLM_TEMPERATURE, + DEFAULT_OLLAMA_HOST, DEFAULT_OLLAMA_TIMEOUT_SECONDS, ) from noteflow.domain.constants.fields import CONTENT @@ -68,12 +69,15 @@ def _get_ollama_settings() -> tuple[str, float, float]: # INTENTIONAL BROAD HANDLER: Fallback for testing environments # - Settings may fail to load in unit tests without full config # - Use sensible defaults for local Ollama server - except Exception: + except Exception as exc: logger.warning( - "Failed to load Ollama settings, using defaults: " - "host=http://localhost:11434, timeout=120s, temperature=0.3" + "ollama_settings_fallback", + error_type=type(exc).__name__, + fallback_host=DEFAULT_OLLAMA_HOST, + fallback_timeout=DEFAULT_OLLAMA_TIMEOUT_SECONDS, + fallback_temperature=DEFAULT_LLM_TEMPERATURE, ) - return ("http://localhost:11434", DEFAULT_OLLAMA_TIMEOUT_SECONDS, DEFAULT_LLM_TEMPERATURE) + return (DEFAULT_OLLAMA_HOST, DEFAULT_OLLAMA_TIMEOUT_SECONDS, DEFAULT_LLM_TEMPERATURE) class OllamaSummarizer(AvailabilityProviderBase): diff --git a/src/noteflow/infrastructure/summarization/template_renderer.py b/src/noteflow/infrastructure/summarization/template_renderer.py index 9a18e43..119d5a6 100644 --- a/src/noteflow/infrastructure/summarization/template_renderer.py +++ b/src/noteflow/infrastructure/summarization/template_renderer.py @@ -8,6 +8,8 @@ from dataclasses import dataclass, field from datetime import datetime from typing import TypeVar +from noteflow.domain.constants.fields import DISPLAY_NAME, ENDED_AT, STATE + @dataclass(frozen=True) class MeetingTemplateContext: @@ -82,10 +84,10 @@ def _stringify(value: str | int | float | None) -> str | None: MEETING_GETTERS: dict[str, Callable[[MeetingTemplateContext], str | None]] = { "id": lambda ctx: ctx.id, "title": lambda ctx: ctx.title, - "state": lambda ctx: ctx.state, + STATE: lambda ctx: ctx.state, "created_at": lambda ctx: ctx.created_at, "started_at": lambda ctx: ctx.started_at, - "ended_at": lambda ctx: ctx.ended_at, + ENDED_AT: lambda ctx: ctx.ended_at, "duration_seconds": lambda ctx: _stringify(ctx.duration_seconds), "duration_minutes": lambda ctx: _stringify(ctx.duration_minutes), "segment_count": lambda ctx: _stringify(ctx.segment_count), @@ -106,7 +108,7 @@ WORKSPACE_GETTERS: dict[str, Callable[[WorkspaceTemplateContext], str | None]] = } USER_GETTERS: dict[str, Callable[[UserTemplateContext], str | None]] = { - "display_name": lambda ctx: ctx.display_name, + DISPLAY_NAME: lambda ctx: ctx.display_name, "email": lambda ctx: ctx.email, } diff --git a/src/noteflow/infrastructure/triggers/app_audio/_sampler.py b/src/noteflow/infrastructure/triggers/app_audio/_sampler.py index 2f402bc..195ba3f 100644 --- a/src/noteflow/infrastructure/triggers/app_audio/_sampler.py +++ b/src/noteflow/infrastructure/triggers/app_audio/_sampler.py @@ -3,7 +3,7 @@ from __future__ import annotations from collections.abc import Mapping, Sequence -from typing import TYPE_CHECKING, cast +from typing import TYPE_CHECKING, Final, cast from noteflow.infrastructure.logging import get_logger @@ -15,6 +15,8 @@ if TYPE_CHECKING: logger = get_logger(__name__) +_PROVIDER_SOUNDDEVICE: Final = "sounddevice" + class SystemOutputSampler: """Best-effort system output sampler using sounddevice.""" @@ -47,7 +49,12 @@ class SystemOutputSampler: self._mark_device_available(loopback_index) return + from noteflow.infrastructure.metrics import get_infrastructure_metrics + self._available = False + get_infrastructure_metrics().record_provider_unavailable( + _PROVIDER_SOUNDDEVICE, "no_loopback_device" + ) logger.warning("No loopback audio device found - app audio detection disabled") def _load_sounddevice(self) -> SoundDeviceModule | None: @@ -126,7 +133,10 @@ class SystemOutputSampler: Args: message: Warning message to log. """ + from noteflow.infrastructure.metrics import get_infrastructure_metrics + self._available = False + get_infrastructure_metrics().record_provider_unavailable(_PROVIDER_SOUNDDEVICE, message) logger.warning(message) def _ensure_stream(self) -> bool: @@ -156,9 +166,14 @@ class SystemOutputSampler: self._stream = stream return True except (OSError, RuntimeError) as exc: + from noteflow.infrastructure.metrics import get_infrastructure_metrics + logger.warning("Failed to start system output capture: %s", exc) self._stream = None self._available = False + get_infrastructure_metrics().record_provider_unavailable( + _PROVIDER_SOUNDDEVICE, f"stream_start_failed: {exc}" + ) return False def read_frames(self, duration_seconds: float) -> NDArray[np.float32] | None: diff --git a/src/noteflow/infrastructure/triggers/foreground_app.py b/src/noteflow/infrastructure/triggers/foreground_app.py index 11b522b..891b2be 100644 --- a/src/noteflow/infrastructure/triggers/foreground_app.py +++ b/src/noteflow/infrastructure/triggers/foreground_app.py @@ -76,13 +76,21 @@ class ForegroundAppProvider: logger.debug("PyWinCtl available for foreground detection") except ImportError: self._available = False - logger.warning("PyWinCtl not installed - foreground detection disabled") + logger.warning( + "pywinctl_unavailable", + error_type="ImportError", + reason="not_installed", + ) # INTENTIONAL BROAD HANDLER: Platform interop via pywinctl # - Different platforms raise different exceptions # - Must gracefully degrade if window detection fails except Exception as e: self._available = False - logger.warning("PyWinCtl unavailable: %s - foreground detection disabled", e) + logger.warning( + "pywinctl_unavailable", + error_type=type(e).__name__, + error_message=str(e), + ) return self._available @@ -98,7 +106,11 @@ class ForegroundAppProvider: window = pywinctl.getActiveWindow() return window.title or None if window else None except Exception as e: - logger.debug("Foreground detection error: %s", e) + logger.debug( + "foreground_detection_error", + error_type=type(e).__name__, + error_message=str(e), + ) return None def _is_suppressed_title(self, title_lower: str) -> bool: diff --git a/tests/application/conftest.py b/tests/application/conftest.py index 8a689c8..79504d2 100644 --- a/tests/application/conftest.py +++ b/tests/application/conftest.py @@ -11,7 +11,7 @@ from uuid import uuid4 import pytest -from noteflow.application.services.webhook_service import WebhookService +from noteflow.application.services.webhooks import WebhookService from noteflow.domain.entities import Meeting, Segment, Summary from noteflow.domain.entities.summary import ActionItem, KeyPoint from noteflow.domain.utils.time import utc_now diff --git a/tests/application/test_asr_config_service.py b/tests/application/test_asr_config_service.py index e6a94b1..8b9cb0b 100644 --- a/tests/application/test_asr_config_service.py +++ b/tests/application/test_asr_config_service.py @@ -16,7 +16,7 @@ from uuid import UUID import pytest -from noteflow.application.services.asr_config_service import ( +from noteflow.application.services.asr_config import ( AsrComputeType, AsrConfigJob, AsrConfigService, @@ -354,6 +354,7 @@ async def test_reconfiguration_failure_keeps_active_engine( ) new_engine = MagicMock() + # Patches must remain active through task completion since job runs in background with ( patch.object(asr_config_service, "_build_engine_for_job", return_value=(new_engine, True)), patch.object(asr_config_service, "_load_model", side_effect=RuntimeError("boom")), @@ -365,13 +366,13 @@ async def test_reconfiguration_failure_keeps_active_engine( has_active_recordings=False, ) - assert job_id is not None, "job should be created even if load fails" - assert error is None, "start_reconfiguration should not return an error on async failure" + assert job_id is not None, "job should be created even if load fails" + assert error is None, "start_reconfiguration should not return an error on async failure" - job = asr_config_service.get_job_status(job_id) - assert job is not None, "job should be retrievable" - assert job.task is not None, "job task should be created" - await job.task + job = asr_config_service.get_job_status(job_id) + assert job is not None, "job should be retrievable" + assert job.task is not None, "job task should be created" + await job.task assert job.status == JOB_STATUS_FAILED, "job should be marked failed on load error" mock_asr_engine.unload.assert_not_called() @@ -390,6 +391,7 @@ async def test_reconfiguration_success_swaps_engine( ) new_engine = MagicMock() + # Patches must remain active through task completion since job runs in background with ( patch.object(asr_config_service, "_build_engine_for_job", return_value=(new_engine, True)), patch.object(asr_config_service, "_load_model", return_value=None), @@ -401,13 +403,13 @@ async def test_reconfiguration_success_swaps_engine( has_active_recordings=False, ) - assert job_id is not None, "job should be created" - assert error is None, "no error should be returned for successful start" + assert job_id is not None, "job should be created" + assert error is None, "no error should be returned for successful start" - job = asr_config_service.get_job_status(job_id) - assert job is not None, "job should be retrievable" - assert job.task is not None, "job task should be created" - await job.task + job = asr_config_service.get_job_status(job_id) + assert job is not None, "job should be retrievable" + assert job.task is not None, "job task should be created" + await job.task assert job.status == JOB_STATUS_COMPLETED, "job should complete successfully" mock_asr_engine.unload.assert_called_once() diff --git a/tests/application/test_auth_service.py b/tests/application/test_auth_service.py index e033e2b..c884aa8 100644 --- a/tests/application/test_auth_service.py +++ b/tests/application/test_auth_service.py @@ -16,7 +16,7 @@ from uuid import UUID, uuid4 import pytest -from noteflow.application.services.auth_service import ( +from noteflow.application.services.auth import ( DEFAULT_USER_ID, DEFAULT_WORKSPACE_ID, AuthResult, @@ -244,7 +244,7 @@ class TestCompleteLogin: mock_auth_oauth_manager.complete_auth.return_value = sample_oauth_tokens with patch( - "noteflow.application.services.auth_service._TokenExchanger.fetch_user_info", + "noteflow.application.services.auth.token_exchanger.TokenExchanger.fetch_user_info", new_callable=AsyncMock, ) as mock_fetch: mock_fetch.return_value = ("test@example.com", "Test User") @@ -281,7 +281,7 @@ class TestCompleteLogin: mock_auth_oauth_manager.complete_auth.return_value = sample_oauth_tokens with patch( - "noteflow.application.services.auth_service._TokenExchanger.fetch_user_info", + "noteflow.application.services.auth.token_exchanger.TokenExchanger.fetch_user_info", new_callable=AsyncMock, ) as mock_fetch: mock_fetch.return_value = ("test@example.com", "New Name") @@ -309,7 +309,7 @@ class TestCompleteLogin: mock_auth_oauth_manager.complete_auth.return_value = sample_oauth_tokens with patch( - "noteflow.application.services.auth_service._TokenExchanger.fetch_user_info", + "noteflow.application.services.auth.token_exchanger.TokenExchanger.fetch_user_info", new_callable=AsyncMock, ) as mock_fetch: mock_fetch.return_value = ("test@example.com", "Test User") diff --git a/tests/application/test_export_service.py b/tests/application/test_export_service.py index 3e30a54..91ce3d2 100644 --- a/tests/application/test_export_service.py +++ b/tests/application/test_export_service.py @@ -9,7 +9,7 @@ from uuid import uuid4 import pytest -from noteflow.application.services.export_service import ExportFormat, ExportService +from noteflow.application.services.export import ExportFormat, ExportService from noteflow.domain.entities import Meeting, Segment from noteflow.domain.value_objects import MeetingId diff --git a/tests/application/test_hf_token_service.py b/tests/application/test_hf_token_service.py index 35f9338..d4ea250 100644 --- a/tests/application/test_hf_token_service.py +++ b/tests/application/test_hf_token_service.py @@ -15,7 +15,7 @@ from unittest.mock import AsyncMock, MagicMock, patch import pytest -from noteflow.application.services.hf_token_service import ( +from noteflow.application.services.huggingface import ( HfTokenService, HfValidationResult, ) @@ -201,10 +201,14 @@ async def test_get_status_no_token_configured( hf_token_service: HfTokenService, mock_preferences: MagicMock, ) -> None: - """get_status returns not configured when no token stored.""" + """get_status returns not configured when no token stored and no env var.""" mock_preferences.get.return_value = None - status = await hf_token_service.get_status() + with patch( + "noteflow.application.services.huggingface.service.get_settings" + ) as mock_settings: + mock_settings.return_value.diarization_hf_token = None + status = await hf_token_service.get_status() assert status.is_configured is False, "is_configured should be False" assert status.is_validated is False, "is_validated should be False" diff --git a/tests/application/test_meeting_service.py b/tests/application/test_meeting_service.py index 95a36f7..928bb51 100644 --- a/tests/application/test_meeting_service.py +++ b/tests/application/test_meeting_service.py @@ -21,15 +21,18 @@ if TYPE_CHECKING: class TestMeetingServiceCreation: """Tests for meeting creation operations.""" - async def test_create_meeting_success(self, mock_uow: MagicMock) -> None: + async def test_create_meeting_success( + self, + sample_meeting: Meeting, + mock_uow: MagicMock, + ) -> None: """Test successful meeting creation.""" - created_meeting = Meeting.create(title="Test Meeting") - mock_uow.meetings.create = AsyncMock(return_value=created_meeting) + mock_uow.meetings.create = AsyncMock(return_value=sample_meeting) service = MeetingService(mock_uow) - result = await service.create_meeting(title="Test Meeting") + result = await service.create_meeting(title=sample_meeting.title) - assert result.title == "Test Meeting" + assert result.title == sample_meeting.title mock_uow.meetings.create.assert_called_once() mock_uow.commit.assert_called_once() diff --git a/tests/application/test_ner_service.py b/tests/application/test_ner_service.py index a123132..1447ff2 100644 --- a/tests/application/test_ner_service.py +++ b/tests/application/test_ner_service.py @@ -7,7 +7,7 @@ from uuid import uuid4 import pytest -from noteflow.application.services.ner_service import ExtractionResult, NerService +from noteflow.application.services.ner import ExtractionResult, NerService from noteflow.domain.entities import Meeting, Segment from noteflow.domain.entities.named_entity import EntityCategory, NamedEntity from noteflow.domain.value_objects import MeetingId @@ -117,7 +117,7 @@ class TestNerServiceExtraction: # Mock feature flag monkeypatch.setattr( - "noteflow.application.services.ner_service.get_feature_flags", + "noteflow.application.services.ner.service.get_feature_flags", lambda: MagicMock(ner_enabled=True), ) @@ -141,7 +141,7 @@ class TestNerServiceExtraction: mock_uow.entities.get_by_meeting = AsyncMock(return_value=cached_entities) monkeypatch.setattr( - "noteflow.application.services.ner_service.get_feature_flags", + "noteflow.application.services.ner.service.get_feature_flags", lambda: MagicMock(ner_enabled=True), ) @@ -169,7 +169,7 @@ class TestNerServiceExtraction: mock_uow.segments.get_by_meeting = AsyncMock(return_value=sample_meeting.segments) monkeypatch.setattr( - "noteflow.application.services.ner_service.get_feature_flags", + "noteflow.application.services.ner.service.get_feature_flags", lambda: MagicMock(ner_enabled=True), ) @@ -196,7 +196,7 @@ class TestNerServiceExtraction: mock_uow.meetings.get = AsyncMock(return_value=None) monkeypatch.setattr( - "noteflow.application.services.ner_service.get_feature_flags", + "noteflow.application.services.ner.service.get_feature_flags", lambda: MagicMock(ner_enabled=True), ) @@ -214,7 +214,7 @@ class TestNerServiceExtraction: ) -> None: """Raises RuntimeError when feature flag is disabled.""" monkeypatch.setattr( - "noteflow.application.services.ner_service.get_feature_flags", + "noteflow.application.services.ner.service.get_feature_flags", lambda: MagicMock(ner_enabled=False), ) @@ -236,7 +236,7 @@ class TestNerServiceExtraction: mock_uow.entities.get_by_meeting = AsyncMock(return_value=[]) monkeypatch.setattr( - "noteflow.application.services.ner_service.get_feature_flags", + "noteflow.application.services.ner.service.get_feature_flags", lambda: MagicMock(ner_enabled=True), ) @@ -348,7 +348,7 @@ class TestNerServiceHelpers: # NER disabled - should be False regardless of engine state monkeypatch.setattr( - "noteflow.application.services.ner_service.get_feature_flags", + "noteflow.application.services.ner.service.get_feature_flags", lambda: MagicMock(ner_enabled=False), ) engine.set_ready(True) @@ -356,7 +356,7 @@ class TestNerServiceHelpers: # NER enabled but engine not ready monkeypatch.setattr( - "noteflow.application.services.ner_service.get_feature_flags", + "noteflow.application.services.ner.service.get_feature_flags", lambda: MagicMock(ner_enabled=True), ) engine.set_ready(False) diff --git a/tests/application/test_retention_service.py b/tests/application/test_retention_service.py index d1b137d..5ec6e87 100644 --- a/tests/application/test_retention_service.py +++ b/tests/application/test_retention_service.py @@ -7,7 +7,7 @@ from unittest.mock import AsyncMock, MagicMock import pytest -from noteflow.application.services.retention_service import RetentionReport, RetentionService +from noteflow.application.services.retention import RetentionReport, RetentionService from noteflow.domain.entities import Meeting diff --git a/tests/application/test_trigger_service.py b/tests/application/test_trigger_service.py index 8702aa4..f1f6b61 100644 --- a/tests/application/test_trigger_service.py +++ b/tests/application/test_trigger_service.py @@ -8,7 +8,7 @@ from dataclasses import dataclass import pytest -from noteflow.application.services.trigger_service import ( +from noteflow.application.services.triggers import ( TriggerService, TriggerServiceSettings, ) diff --git a/tests/application/test_webhook_service.py b/tests/application/test_webhook_service.py index 9833a52..12cd7f8 100644 --- a/tests/application/test_webhook_service.py +++ b/tests/application/test_webhook_service.py @@ -7,7 +7,7 @@ from uuid import uuid4 import pytest -from noteflow.application.services.webhook_service import WebhookService +from noteflow.application.services.webhooks import WebhookService from noteflow.domain.entities import Meeting from noteflow.domain.utils.time import utc_now from noteflow.domain.webhooks import ( diff --git a/tests/domain/conftest.py b/tests/domain/conftest.py index 100a5a5..39483cb 100644 --- a/tests/domain/conftest.py +++ b/tests/domain/conftest.py @@ -6,10 +6,14 @@ meetings with segments for capacity and edge case testing. from __future__ import annotations +from uuid import uuid4 + import pytest +from noteflow.domain.entities.annotation import Annotation from noteflow.domain.entities.meeting import Meeting from noteflow.domain.entities.segment import Segment +from noteflow.domain.value_objects import AnnotationId, AnnotationType, MeetingId def create_segment_at_index(index: int) -> Segment: @@ -61,3 +65,25 @@ def _populate_segments(meeting: Meeting, count: int) -> None: def meeting_with_100_segments() -> Meeting: """Create a meeting with 100 segments for capacity testing.""" return create_meeting_with_many_segments(100) + + +@pytest.fixture +def annotation_id() -> AnnotationId: + """Generate a test AnnotationId.""" + return AnnotationId(uuid4()) + + +@pytest.fixture +def sample_annotation( + annotation_id: AnnotationId, + meeting_id: MeetingId, +) -> Annotation: + """Valid Annotation for testing basic operations.""" + return Annotation( + id=annotation_id, + meeting_id=meeting_id, + annotation_type=AnnotationType.NOTE, + text="Sample annotation text", + start_time=1.0, + end_time=2.0, + ) diff --git a/tests/domain/test_annotation.py b/tests/domain/test_annotation.py index 4b43933..6b92dd5 100644 --- a/tests/domain/test_annotation.py +++ b/tests/domain/test_annotation.py @@ -16,20 +16,11 @@ TWO_HOURS_SECONDS = 7200.0 class TestAnnotation: """Tests for Annotation entity.""" - def test_annotation_valid(self) -> None: - """Annotation can be created with valid fields.""" - annotation = Annotation( - id=AnnotationId(uuid4()), - meeting_id=MeetingId(uuid4()), - annotation_type=AnnotationType.NOTE, - text="Important point", - start_time=1.0, - end_time=2.0, - ) - - assert annotation.text == "Important point", "text should match provided value" - assert annotation.duration == 1.0, "duration should be end - start" - assert annotation.has_segments() is False, "no segment_ids means has_segments is False" + def test_annotation_valid(self, sample_annotation: Annotation) -> None: + """Annotation fixture has expected properties.""" + assert sample_annotation.text == "Sample annotation text", "text should match fixture" + assert sample_annotation.duration == 1.0, "duration should be end - start" + assert sample_annotation.has_segments() is False, "no segment_ids means has_segments is False" def test_annotation_invalid_times_raises(self) -> None: """Annotation raises when end_time < start_time.""" diff --git a/tests/domain/test_meeting.py b/tests/domain/test_meeting.py index 0a95c23..6280292 100644 --- a/tests/domain/test_meeting.py +++ b/tests/domain/test_meeting.py @@ -117,11 +117,17 @@ class TestMeetingStateTransitions: with pytest.raises(ValueError, match="Cannot stop recording"): meeting.stop_recording() - def test_stop_recording_from_created_raises(self) -> None: - """Test stopping recording from CREATED state raises.""" + def test_stop_recording_from_created_succeeds(self) -> None: + """Test stopping recording from CREATED state transitions to STOPPED. + + This is a valid scenario for meetings that never started recording + but need to be closed. + """ meeting = Meeting.create() - with pytest.raises(ValueError, match="Cannot stop recording"): - meeting.stop_recording() + assert meeting.state == MeetingState.CREATED, "meeting should start in CREATED state" + meeting.stop_recording() + assert meeting.state == MeetingState.STOPPED, "state should transition to STOPPED" + assert meeting.ended_at is not None, "ended_at should be set when stopping" def test_complete_from_stopped(self) -> None: """Test completing meeting from STOPPED state.""" diff --git a/tests/domain/test_named_entity.py b/tests/domain/test_named_entity.py index 4317ca5..135b727 100644 --- a/tests/domain/test_named_entity.py +++ b/tests/domain/test_named_entity.py @@ -130,21 +130,19 @@ class TestNamedEntityCreate: ) assert entity.segment_ids == [1, 2, 3], f"expected deduplicated/sorted segment_ids [1, 2, 3], got {entity.segment_ids}" - def test_create_empty_text_raises(self) -> None: - """Create with empty text raises ValueError.""" + @pytest.mark.parametrize( + "invalid_text", + [ + pytest.param("", id="empty-text"), + pytest.param(" ", id="whitespace-only"), + pytest.param("\t\n", id="tabs-and-newlines"), + ], + ) + def test_create_invalid_text_raises(self, invalid_text: str) -> None: + """Create with empty or whitespace-only text raises ValueError.""" with pytest.raises(ValueError, match="Entity text cannot be empty"): NamedEntity.create( - text="", - category=EntityCategory.PERSON, - segment_ids=[0], - confidence=0.9, - ) - - def test_create_whitespace_only_text_raises(self) -> None: - """Create with whitespace-only text raises ValueError.""" - with pytest.raises(ValueError, match="Entity text cannot be empty"): - NamedEntity.create( - text=" ", + text=invalid_text, category=EntityCategory.PERSON, segment_ids=[0], confidence=0.9, diff --git a/tests/domain/test_value_objects.py b/tests/domain/test_value_objects.py index d066004..f5a9381 100644 --- a/tests/domain/test_value_objects.py +++ b/tests/domain/test_value_objects.py @@ -21,7 +21,7 @@ class TestMeetingState: # CREATED transitions (MeetingState.CREATED, MeetingState.RECORDING, True), (MeetingState.CREATED, MeetingState.ERROR, True), - (MeetingState.CREATED, MeetingState.STOPPED, False), + (MeetingState.CREATED, MeetingState.STOPPED, True), # RECORDING transitions (must go through STOPPING) (MeetingState.RECORDING, MeetingState.STOPPING, True), (MeetingState.RECORDING, MeetingState.STOPPED, False), diff --git a/tests/grpc/conftest.py b/tests/grpc/conftest.py index 0019284..883be9b 100644 --- a/tests/grpc/conftest.py +++ b/tests/grpc/conftest.py @@ -14,6 +14,8 @@ from uuid import uuid4 import pytest +from noteflow.domain.entities import Meeting +from noteflow.domain.value_objects import MeetingId, MeetingState from noteflow.domain.webhooks import ( DeliveryResult, WebhookConfig, @@ -91,3 +93,40 @@ def sample_webhook_delivery(sample_webhook_config: WebhookConfig) -> WebhookDeli payload={"event": "meeting.completed", "meeting_id": str(uuid4())}, result=result, ) + + +def create_test_meeting( + meeting_id: MeetingId | None = None, + title: str = "Test Meeting", + state: MeetingState = MeetingState.CREATED, +) -> Meeting: + """Factory for creating test Meeting entities with specific state. + + Transitions through valid states to reach target state. + + Args: + meeting_id: Optional meeting ID to assign. + title: Meeting title (default "Test Meeting"). + state: Target meeting state (default CREATED). + + Returns: + Meeting entity in the specified state. + """ + meeting = Meeting.create(title=title) + if meeting_id is not None: + meeting.id = meeting_id + + # Transition through valid states to reach target + if state in { + MeetingState.RECORDING, + MeetingState.STOPPING, + MeetingState.STOPPED, + MeetingState.COMPLETED, + }: + meeting.start_recording() + if state in {MeetingState.STOPPING, MeetingState.STOPPED, MeetingState.COMPLETED}: + meeting.stop_recording() + if state == MeetingState.COMPLETED: + meeting.complete() + + return meeting diff --git a/tests/grpc/proto_types.py b/tests/grpc/proto_types.py index 6824d8b..93eb05f 100644 --- a/tests/grpc/proto_types.py +++ b/tests/grpc/proto_types.py @@ -10,7 +10,7 @@ from collections.abc import Mapping, Sequence from typing import Protocol from unittest.mock import MagicMock -from noteflow.grpc._mixins._types import GrpcContext +from noteflow.grpc.mixins._types import GrpcContext class CreateMeetingRequestProto(Protocol): diff --git a/tests/grpc/test_annotation_mixin.py b/tests/grpc/test_annotation_mixin.py index 950de6e..bff965b 100644 --- a/tests/grpc/test_annotation_mixin.py +++ b/tests/grpc/test_annotation_mixin.py @@ -20,8 +20,8 @@ import pytest from noteflow.domain.entities import Annotation from noteflow.domain.value_objects import AnnotationId, AnnotationType, MeetingId -from noteflow.grpc._mixins._types import GrpcContext -from noteflow.grpc._mixins.annotation import AnnotationMixin +from noteflow.grpc.mixins._types import GrpcContext +from noteflow.grpc.mixins.annotation import AnnotationMixin from noteflow.grpc.proto import noteflow_pb2 # Test constants for annotation timestamps and time ranges diff --git a/tests/grpc/test_chunk_sequence_tracking.py b/tests/grpc/test_chunk_sequence_tracking.py index 939c6ae..13c6e3e 100644 --- a/tests/grpc/test_chunk_sequence_tracking.py +++ b/tests/grpc/test_chunk_sequence_tracking.py @@ -9,9 +9,9 @@ from unittest.mock import MagicMock import pytest -from noteflow.grpc._mixins.converters import create_ack_update -from noteflow.grpc._mixins.streaming._processing._chunk_tracking import track_chunk_sequence -from noteflow.grpc._mixins.streaming._processing._constants import ACK_CHUNK_INTERVAL +from noteflow.grpc.mixins.converters import create_ack_update +from noteflow.grpc.mixins.streaming._processing._chunk_tracking import track_chunk_sequence +from noteflow.grpc.mixins.streaming._processing._constants import ACK_CHUNK_INTERVAL from noteflow.grpc.proto import noteflow_pb2 # Test constants for sequence tracking @@ -138,7 +138,7 @@ class TestTrackChunkSequence: @pytest.mark.parametrize( ("chunk_count", "expects_ack"), [ - pytest.param(1, False, id="chunk_1_no_ack"), + pytest.param(1, True, id="chunk_1_emits_immediate_ack"), pytest.param(ACK_CHUNK_INTERVAL // 2, False, id="midpoint_no_ack"), pytest.param(ACK_CHUNK_INTERVAL - 1, False, id="one_before_interval_no_ack"), pytest.param(ACK_CHUNK_INTERVAL, True, id="at_interval_emits_ack"), @@ -147,7 +147,7 @@ class TestTrackChunkSequence: def test_ack_emission_at_chunk_count( self, mock_host: MagicMock, chunk_count: int, expects_ack: bool ) -> None: - """Verify ack emission depends on reaching ACK_CHUNK_INTERVAL.""" + """Verify ack emission: first chunk gets immediate ack, then at interval.""" meeting_id = "test-meeting" _send_chunks_up_to(mock_host, meeting_id, chunk_count - 1) diff --git a/tests/grpc/test_client_result.py b/tests/grpc/test_client_result.py new file mode 100644 index 0000000..0d62750 --- /dev/null +++ b/tests/grpc/test_client_result.py @@ -0,0 +1,215 @@ +"""Tests for ClientResult type and error handling utilities.""" + +from __future__ import annotations + +from unittest.mock import MagicMock + +import grpc +import pytest + +from noteflow.grpc.client_mixins._error_handling import ( + grpc_error_to_result, + not_connected_error, +) +from noteflow.grpc.types import ClientErrorCode, ClientResult, Err, Ok + + +class TestOkResult: + """Test successful result creation.""" + + def test_ok_creates_successful_result(self) -> None: + """Ok() creates result with success=True.""" + result = Ok("test_value") + + assert ( + result.success + and not result.failed + and result.value == "test_value" + and result.error is None + ), f"Ok() should create successful result: success={result.success}, value={result.value}" + + def test_ok_unwrap_returns_value(self) -> None: + """unwrap() returns value for successful result.""" + result: ClientResult[str] = Ok("test_value") + + assert result.unwrap() == "test_value" + + def test_ok_unwrap_or_returns_value(self) -> None: + """unwrap_or() returns value for successful result.""" + result: ClientResult[str] = Ok("test_value") + + assert result.unwrap_or("default") == "test_value" + + +class TestErrResult: + """Test failed result creation.""" + + def test_err_creates_failed_result(self) -> None: + """Err() creates result with failed=True.""" + result: ClientResult[str] = Err( + code=ClientErrorCode.NOT_FOUND, message="Not found" + ) + + assert ( + result.failed + and not result.success + and result.value is None + and result.error is not None + and result.error.code == ClientErrorCode.NOT_FOUND + and result.error.message == "Not found" + ), f"Err() should create failed result: {result}" + + def test_err_unwrap_raises(self) -> None: + """unwrap() raises for failed result.""" + result: ClientResult[str] = Err( + code=ClientErrorCode.INTERNAL, message="Internal error" + ) + + with pytest.raises(RuntimeError, match="Client operation failed: Internal error"): + result.unwrap() + + def test_err_unwrap_or_returns_default(self) -> None: + """unwrap_or() returns default for failed result.""" + result: ClientResult[str] = Err( + code=ClientErrorCode.UNAVAILABLE, message="Service unavailable" + ) + + assert result.unwrap_or("default") == "default" + + +class TestClientError: + """Test ClientError properties.""" + + @pytest.mark.parametrize( + "code", + [ + ClientErrorCode.UNAVAILABLE, + ClientErrorCode.DEADLINE_EXCEEDED, + ClientErrorCode.INTERNAL, + ], + ) + def test_retryable_errors(self, code: ClientErrorCode) -> None: + """is_retryable identifies transient errors.""" + result: ClientResult[str] = Err(code=code, message="Error") + + assert ( + result.error is not None and result.error.is_retryable + ), f"{code} should be retryable" + + @pytest.mark.parametrize( + "code", + [ + ClientErrorCode.NOT_FOUND, + ClientErrorCode.INVALID_ARGUMENT, + ClientErrorCode.PERMISSION_DENIED, + ], + ) + def test_non_retryable_errors(self, code: ClientErrorCode) -> None: + """is_retryable correctly identifies non-retryable errors.""" + result: ClientResult[str] = Err(code=code, message="Error") + + assert ( + result.error is not None and not result.error.is_retryable + ), f"{code} should not be retryable" + + def test_not_found_property(self) -> None: + """is_not_found correctly identifies NOT_FOUND errors.""" + result: ClientResult[str] = Err( + code=ClientErrorCode.NOT_FOUND, message="Not found" + ) + + assert ( + result.error is not None and result.error.is_not_found + ), "NOT_FOUND error should have is_not_found=True" + + def test_not_found_property_false_for_other_codes(self) -> None: + """is_not_found returns False for non-NOT_FOUND errors.""" + result: ClientResult[str] = Err( + code=ClientErrorCode.INTERNAL, message="Internal error" + ) + + assert ( + result.error is not None and not result.error.is_not_found + ), "INTERNAL error should have is_not_found=False" + + +class TestGrpcErrorConversion: + """Test conversion of gRPC errors to ClientResult.""" + + @pytest.mark.parametrize( + ("grpc_code", "expected_client_code"), + [ + (grpc.StatusCode.NOT_FOUND, ClientErrorCode.NOT_FOUND), + (grpc.StatusCode.INVALID_ARGUMENT, ClientErrorCode.INVALID_ARGUMENT), + (grpc.StatusCode.DEADLINE_EXCEEDED, ClientErrorCode.DEADLINE_EXCEEDED), + (grpc.StatusCode.ALREADY_EXISTS, ClientErrorCode.ALREADY_EXISTS), + (grpc.StatusCode.PERMISSION_DENIED, ClientErrorCode.PERMISSION_DENIED), + (grpc.StatusCode.FAILED_PRECONDITION, ClientErrorCode.FAILED_PRECONDITION), + (grpc.StatusCode.UNIMPLEMENTED, ClientErrorCode.UNIMPLEMENTED), + (grpc.StatusCode.INTERNAL, ClientErrorCode.INTERNAL), + (grpc.StatusCode.UNAVAILABLE, ClientErrorCode.UNAVAILABLE), + (grpc.StatusCode.RESOURCE_EXHAUSTED, ClientErrorCode.UNAVAILABLE), + ], + ) + def test_grpc_status_code_mapping( + self, grpc_code: grpc.StatusCode, expected_client_code: ClientErrorCode + ) -> None: + """grpc_error_to_result correctly maps gRPC status codes.""" + mock_error = MagicMock() + mock_error.code = MagicMock(return_value=grpc_code) + mock_error.details = MagicMock(return_value="Test error") + + result: ClientResult[str] = grpc_error_to_result(mock_error, "test_operation") + + assert ( + result.failed + and result.error is not None + and result.error.code == expected_client_code + and result.error.message == "Test error" + and result.error.grpc_status == grpc_code + ), f"gRPC {grpc_code} should map to {expected_client_code}" + + def test_grpc_error_without_details(self) -> None: + """grpc_error_to_result handles errors without details.""" + mock_error = MagicMock() + mock_error.code = MagicMock(return_value=grpc.StatusCode.INTERNAL) + mock_error.details = MagicMock(return_value=None) + mock_error.__str__ = MagicMock(return_value="Generic error") + + result: ClientResult[str] = grpc_error_to_result(mock_error, "test_operation") + + assert ( + result.failed + and result.error is not None + and result.error.message == "Generic error" + ), "Error without details should use str(error) as message" + + def test_grpc_error_unknown_status(self) -> None: + """grpc_error_to_result maps unknown status to UNKNOWN.""" + mock_error = MagicMock() + mock_error.code = MagicMock(return_value=grpc.StatusCode.CANCELLED) + mock_error.details = MagicMock(return_value="Operation cancelled") + + result: ClientResult[str] = grpc_error_to_result(mock_error, "test_operation") + + assert ( + result.failed + and result.error is not None + and result.error.code == ClientErrorCode.UNKNOWN + ), "Unmapped gRPC status should become UNKNOWN" + + +class TestNotConnectedError: + """Test not_connected_error utility.""" + + def test_creates_not_connected_error(self) -> None: + """not_connected_error creates NOT_CONNECTED error.""" + result: ClientResult[str] = not_connected_error("test_operation") + + assert ( + result.failed + and result.error is not None + and result.error.code == ClientErrorCode.NOT_CONNECTED + and result.error.message == "Not connected to server" + and result.error.grpc_status is None + ), "not_connected_error should create NOT_CONNECTED error" diff --git a/tests/grpc/test_cloud_consent.py b/tests/grpc/test_cloud_consent.py index 5c96491..9cd81dd 100644 --- a/tests/grpc/test_cloud_consent.py +++ b/tests/grpc/test_cloud_consent.py @@ -15,7 +15,7 @@ from noteflow.application.services.summarization import ( SummarizationService, SummarizationServiceSettings, ) -from noteflow.grpc._config import ServicesConfig +from noteflow.grpc.config.config import ServicesConfig from noteflow.grpc.proto import noteflow_pb2 from noteflow.grpc.service import NoteFlowServicer diff --git a/tests/grpc/test_congestion_tracking.py b/tests/grpc/test_congestion_tracking.py index 7994558..c354c44 100644 --- a/tests/grpc/test_congestion_tracking.py +++ b/tests/grpc/test_congestion_tracking.py @@ -11,11 +11,11 @@ from unittest.mock import MagicMock import pytest -from noteflow.grpc._mixins.streaming._processing._congestion import ( +from noteflow.grpc.mixins.streaming._processing._congestion import ( calculate_congestion_info, decrement_pending_chunks, ) -from noteflow.grpc._mixins.streaming._processing._constants import ( +from noteflow.grpc.mixins.streaming._processing._constants import ( PROCESSING_DELAY_THRESHOLD_MS, QUEUE_DEPTH_THRESHOLD, ) @@ -183,7 +183,7 @@ class TestDecrementPendingChunks: decrement_pending_chunks(congestion_host, meeting_id) def test_decrements_pending_count(self, congestion_host: MagicMock) -> None: - """Verify decrement reduces pending chunk count.""" + """Verify decrement reduces pending chunk count by 1.""" meeting_id = "test-meeting" initial_pending = 15 congestion_host.pending_chunks[meeting_id] = initial_pending @@ -193,20 +193,21 @@ class TestDecrementPendingChunks: decrement_pending_chunks(congestion_host, meeting_id) - assert congestion_host.pending_chunks[meeting_id] == 10, "should decrement by 5" + expected_remaining = initial_pending - 1 + assert congestion_host.pending_chunks[meeting_id] == expected_remaining, "should decrement by 1" def test_clamps_to_zero(self, congestion_host: MagicMock) -> None: """Verify pending chunks don't go negative.""" meeting_id = "test-meeting" - congestion_host.pending_chunks[meeting_id] = 2 # Less than ACK_CHUNK_INTERVAL - congestion_host.chunk_receipt_times[meeting_id] = deque([1.0, 2.0]) + congestion_host.pending_chunks[meeting_id] = 0 # Already at zero + congestion_host.chunk_receipt_times[meeting_id] = deque([]) decrement_pending_chunks(congestion_host, meeting_id) assert congestion_host.pending_chunks[meeting_id] == 0, "should clamp to 0" def test_clears_receipt_times_on_decrement(self, congestion_host: MagicMock) -> None: - """Verify old receipt times are cleared when processing completes.""" + """Verify oldest receipt time is cleared when processing completes.""" meeting_id = "test-meeting" congestion_host.pending_chunks[meeting_id] = 8 congestion_host.chunk_receipt_times[meeting_id] = deque( @@ -216,5 +217,5 @@ class TestDecrementPendingChunks: decrement_pending_chunks(congestion_host, meeting_id) remaining = len(congestion_host.chunk_receipt_times[meeting_id]) - # 8 - ACK_CHUNK_INTERVAL (5) = 3 remaining - assert remaining == 3, "should have 3 receipt times remaining after decrementing 5" + # 8 - 1 = 7 remaining (one popleft per call) + assert remaining == 7, "should have 7 receipt times remaining after decrementing 1" diff --git a/tests/grpc/test_diarization_lifecycle.py b/tests/grpc/test_diarization_lifecycle.py index 09058a8..2421baf 100644 --- a/tests/grpc/test_diarization_lifecycle.py +++ b/tests/grpc/test_diarization_lifecycle.py @@ -15,7 +15,7 @@ import grpc import pytest from noteflow.domain.entities import Meeting -from noteflow.grpc._config import ServicesConfig +from noteflow.grpc.config.config import ServicesConfig from noteflow.grpc.proto import noteflow_pb2 from noteflow.grpc.service import NoteFlowServicer from noteflow.infrastructure.diarization import DiarizationEngine @@ -210,14 +210,17 @@ class TestDatabaseRequirement: @pytest.mark.asyncio async def test_cancel_requires_database(self, servicer: NoteFlowServicer) -> None: - """CancelDiarizationJob returns error when database unavailable.""" + """CancelDiarizationJob aborts when database unavailable.""" context = _MockGrpcContext() - cancel = cast(_CancelDiarizationJobCallable, servicer.CancelDiarizationJob) - response = await cancel( - noteflow_pb2.CancelDiarizationJobRequest(job_id="any-job-id"), - context, - ) + # Should abort because no DB support + with pytest.raises(_AbortCalled, match="database") as exc_info: + cancel = cast(_CancelDiarizationJobCallable, servicer.CancelDiarizationJob) + await cancel( + noteflow_pb2.CancelDiarizationJobRequest(job_id="any-job-id"), + context, + ) - assert response.success is False, "Cancel should fail without database" - assert "database" in response.error_message.lower(), "Error should mention database" + # Verify abort was called with UNIMPLEMENTED (no DB support) + assert exc_info.value.code == grpc.StatusCode.UNIMPLEMENTED, "Should abort with UNIMPLEMENTED" + assert "database" in exc_info.value.details.lower(), "Error should mention database" diff --git a/tests/grpc/test_diarization_mixin.py b/tests/grpc/test_diarization_mixin.py index 53b9f2a..a8dfd87 100644 --- a/tests/grpc/test_diarization_mixin.py +++ b/tests/grpc/test_diarization_mixin.py @@ -20,7 +20,7 @@ from noteflow.domain.entities import Meeting from noteflow.domain.entities.segment import Segment from noteflow.domain.utils import utc_now from noteflow.domain.value_objects import MeetingId -from noteflow.grpc._config import ServicesConfig +from noteflow.grpc.config.config import ServicesConfig from noteflow.grpc.meeting_store import MeetingStore from noteflow.grpc.proto import noteflow_pb2 from noteflow.grpc.service import NoteFlowServicer @@ -878,14 +878,16 @@ class TestCancelDiarizationJobStates: """Nonexistent job cannot be cancelled.""" context = _MockGrpcContext() - response = await _call_cancel( - diarization_servicer_with_db, - noteflow_pb2.CancelDiarizationJobRequest(job_id="nonexistent-job"), - context, - ) + with pytest.raises(AssertionError, match="abort called"): + await _call_cancel( + diarization_servicer_with_db, + noteflow_pb2.CancelDiarizationJobRequest(job_id="nonexistent-job"), + context, + ) - assert response.success is False, "Cancelling nonexistent job should fail" - assert "not found" in response.error_message.lower(), "Error message should mention 'not found'" + assert context.abort_code == grpc.StatusCode.NOT_FOUND, "Should abort with NOT_FOUND" + assert context.abort_details is not None, "Abort details should be set" + assert "not found" in context.abort_details.lower(), "Error message should mention 'not found'" @pytest.mark.asyncio async def test_cancel_completed_job_fails( @@ -904,11 +906,11 @@ class TestCancelDiarizationJobStates: ) context = _MockGrpcContext() - response = await _call_cancel( - diarization_servicer_with_db, - noteflow_pb2.CancelDiarizationJobRequest(job_id=job_id), - context, - ) + with pytest.raises(AssertionError, match="abort called"): + await _call_cancel( + diarization_servicer_with_db, + noteflow_pb2.CancelDiarizationJobRequest(job_id=job_id), + context, + ) - assert response.success is False, "Cancelling completed job should fail" - assert response.status == JOB_STATUS_COMPLETED, "Completed job status should remain COMPLETED" + assert context.abort_code == grpc.StatusCode.FAILED_PRECONDITION, "Should abort with FAILED_PRECONDITION" diff --git a/tests/grpc/test_diarization_refine.py b/tests/grpc/test_diarization_refine.py index 4b7766b..df14fec 100644 --- a/tests/grpc/test_diarization_refine.py +++ b/tests/grpc/test_diarization_refine.py @@ -7,7 +7,7 @@ from typing import Protocol, cast import grpc import pytest -from noteflow.grpc._config import ServicesConfig +from noteflow.grpc.config.config import ServicesConfig from noteflow.grpc.proto import noteflow_pb2 from noteflow.grpc.service import NoteFlowServicer from noteflow.infrastructure.diarization import DiarizationEngine diff --git a/tests/grpc/test_entities_mixin.py b/tests/grpc/test_entities_mixin.py index ad5309f..a1ed088 100644 --- a/tests/grpc/test_entities_mixin.py +++ b/tests/grpc/test_entities_mixin.py @@ -14,11 +14,11 @@ from uuid import uuid4 import pytest -from noteflow.application.services.ner_service import ExtractionResult +from noteflow.application.services.ner import ExtractionResult from noteflow.domain.entities.named_entity import EntityCategory, NamedEntity from noteflow.domain.value_objects import MeetingId -from noteflow.grpc._mixins._types import GrpcContext -from noteflow.grpc._mixins.entities import EntitiesMixin +from noteflow.grpc.mixins._types import GrpcContext +from noteflow.grpc.mixins.entities import EntitiesMixin from noteflow.grpc.proto import noteflow_pb2 if TYPE_CHECKING: @@ -65,6 +65,10 @@ class MockServicerHost(EntitiesMixin): self._entities_repo = entities_repo self._meetings_repo = meetings_repo or AsyncMock() self.ner_service = ner_service + # Required by log_model_status in _model_status.py + self.asr_engine = None + self.diarization_engine = None + self.summarization_service = None def create_repository_provider(self) -> MockRepositoryProvider: """Create mock repository provider context manager.""" diff --git a/tests/grpc/test_export_mixin.py b/tests/grpc/test_export_mixin.py index 123ddb2..07a61b6 100644 --- a/tests/grpc/test_export_mixin.py +++ b/tests/grpc/test_export_mixin.py @@ -17,11 +17,11 @@ from uuid import uuid4 import pytest -from noteflow.application.services.export_service import ExportFormat +from noteflow.application.services.export import ExportFormat from noteflow.domain.entities import Meeting, Segment from noteflow.domain.value_objects import MeetingId -from noteflow.grpc._mixins._types import GrpcContext -from noteflow.grpc._mixins.export import ExportMixin +from noteflow.grpc.mixins._types import GrpcContext +from noteflow.grpc.mixins.export import ExportMixin from noteflow.grpc.proto import noteflow_pb2 @@ -166,7 +166,7 @@ class TestExportTranscriptMarkdown: ) with patch( - "noteflow.grpc._mixins.export.ExportService" + "noteflow.grpc.mixins.export.ExportService" ) as mock_export_service_cls: mock_service = MagicMock() mock_service.export_transcript = AsyncMock( @@ -198,7 +198,7 @@ class TestExportTranscriptMarkdown: meeting_id=str(meeting_id), format=noteflow_pb2.EXPORT_FORMAT_MARKDOWN, ) - with patch("noteflow.grpc._mixins.export.ExportService") as mock_cls: + with patch("noteflow.grpc.mixins.export.ExportService") as mock_cls: mock_cls.return_value.export_transcript = AsyncMock(return_value=expected) response = await export_servicer.ExportTranscript(request, mock_grpc_context) @@ -224,7 +224,7 @@ class TestExportTranscriptMarkdown: ) with patch( - "noteflow.grpc._mixins.export.ExportService" + "noteflow.grpc.mixins.export.ExportService" ) as mock_export_service_cls: mock_service = MagicMock() mock_service.export_transcript = AsyncMock(return_value="# Meeting") @@ -262,7 +262,7 @@ class TestExportTranscriptHtml: html_content = "

Design Review

" with patch( - "noteflow.grpc._mixins.export.ExportService" + "noteflow.grpc.mixins.export.ExportService" ) as mock_export_service_cls: mock_service = MagicMock() mock_service.export_transcript = AsyncMock(return_value=html_content) @@ -293,7 +293,7 @@ class TestExportTranscriptHtml: meeting_id=str(meeting_id), format=noteflow_pb2.EXPORT_FORMAT_HTML, ) - with patch("noteflow.grpc._mixins.export.ExportService") as mock_cls: + with patch("noteflow.grpc.mixins.export.ExportService") as mock_cls: mock_cls.return_value.export_transcript = AsyncMock(return_value=html_content) response = await export_servicer.ExportTranscript(request, mock_grpc_context) @@ -327,7 +327,7 @@ class TestExportTranscriptPdf: expected_base64 = base64.b64encode(pdf_bytes).decode("ascii") with patch( - "noteflow.grpc._mixins.export.ExportService" + "noteflow.grpc.mixins.export.ExportService" ) as mock_export_service_cls: mock_service = MagicMock() mock_service.export_transcript = AsyncMock(return_value=pdf_bytes) @@ -360,7 +360,7 @@ class TestExportTranscriptPdf: original_pdf_bytes = b"%PDF-1.4\nReal PDF binary data here" with patch( - "noteflow.grpc._mixins.export.ExportService" + "noteflow.grpc.mixins.export.ExportService" ) as mock_export_service_cls: mock_service = MagicMock() mock_service.export_transcript = AsyncMock(return_value=original_pdf_bytes) @@ -389,7 +389,7 @@ class TestExportTranscriptMeetingNotFound: ) with patch( - "noteflow.grpc._mixins.export.ExportService" + "noteflow.grpc.mixins.export.ExportService" ) as mock_export_service_cls: mock_service = MagicMock() mock_service.export_transcript = AsyncMock( @@ -460,7 +460,7 @@ class TestExportTranscriptWithSegments: meeting_id=str(meeting_id), format=noteflow_pb2.EXPORT_FORMAT_MARKDOWN, ) - with patch("noteflow.grpc._mixins.export.ExportService") as mock_cls: + with patch("noteflow.grpc.mixins.export.ExportService") as mock_cls: mock_cls.return_value.export_transcript = AsyncMock(return_value=expected) response = await export_servicer.ExportTranscript(request, mock_grpc_context) @@ -486,7 +486,7 @@ class TestExportTranscriptWithSegments: ) with patch( - "noteflow.grpc._mixins.export.ExportService" + "noteflow.grpc.mixins.export.ExportService" ) as mock_export_service_cls: mock_service = MagicMock() mock_service.export_transcript = AsyncMock( @@ -520,7 +520,7 @@ class TestExportTranscriptWithSegments: meeting_id=str(meeting_id), format=noteflow_pb2.EXPORT_FORMAT_HTML, ) - with patch("noteflow.grpc._mixins.export.ExportService") as mock_cls: + with patch("noteflow.grpc.mixins.export.ExportService") as mock_cls: mock_cls.return_value.export_transcript = AsyncMock(return_value=long_content) response = await export_servicer.ExportTranscript(request, mock_grpc_context) @@ -560,7 +560,7 @@ class TestExportFormatMetadata: request = noteflow_pb2.ExportTranscriptRequest(meeting_id=str(meeting_id), format=proto_format) - with patch("noteflow.grpc._mixins.export.ExportService") as mock_cls: + with patch("noteflow.grpc.mixins.export.ExportService") as mock_cls: mock_cls.return_value.export_transcript = AsyncMock(return_value=export_result) response = await export_servicer.ExportTranscript(request, mock_grpc_context) diff --git a/tests/grpc/test_generate_summary.py b/tests/grpc/test_generate_summary.py index 0916e44..871ef09 100644 --- a/tests/grpc/test_generate_summary.py +++ b/tests/grpc/test_generate_summary.py @@ -16,7 +16,7 @@ from noteflow.application.services.summarization import ( from noteflow.domain.entities import Segment from noteflow.domain.summarization import ProviderUnavailableError from noteflow.domain.value_objects import MeetingId -from noteflow.grpc._config import ServicesConfig +from noteflow.grpc.config.config import ServicesConfig from noteflow.grpc.proto import noteflow_pb2 from noteflow.grpc.service import NoteFlowServicer diff --git a/tests/grpc/test_identity_mixin.py b/tests/grpc/test_identity_mixin.py index 1ae6277..97fed82 100644 --- a/tests/grpc/test_identity_mixin.py +++ b/tests/grpc/test_identity_mixin.py @@ -24,8 +24,8 @@ from noteflow.domain.identity.entities import ( WorkspaceSettings, ) from noteflow.domain.identity.roles import WorkspaceRole -from noteflow.grpc._mixins._types import GrpcContext -from noteflow.grpc._mixins.identity import IdentityMixin +from noteflow.grpc.mixins._types import GrpcContext +from noteflow.grpc.mixins.identity import IdentityMixin from noteflow.grpc.proto import noteflow_pb2 if TYPE_CHECKING: diff --git a/tests/grpc/test_interceptors.py b/tests/grpc/test_interceptors.py index 55d1d1a..51b0c4e 100644 --- a/tests/grpc/test_interceptors.py +++ b/tests/grpc/test_interceptors.py @@ -154,15 +154,19 @@ class TestIdentityInterceptor: interceptor = IdentityInterceptor() # No metadata - missing required x-request-id details = create_handler_call_details(metadata=[]) - continuation = create_mock_continuation() + original_handler = create_mock_handler() + continuation = create_mock_continuation(original_handler) handler = await interceptor.intercept_service(continuation, details) typed_handler = cast(_UnaryUnaryHandler, handler) - # Handler should be a rejection handler, not the original + # Handler should be a rejection handler wrapping the original assert typed_handler.unary_unary is not None, "handler should have unary_unary" - # Continuation should NOT have been called - continuation.assert_not_called() + # Continuation is called to get handler type info, but the returned handler + # is a rejection wrapper that will abort with UNAUTHENTICATED + continuation.assert_called_once() + # The returned handler should NOT be the original handler + assert typed_handler.unary_unary is not original_handler.unary_unary, "should return rejection handler, not original" @pytest.mark.asyncio async def test_reject_handler_aborts_with_unauthenticated(self) -> None: @@ -219,7 +223,7 @@ class TestRequestLoggingInterceptor: # Set request_id for logging context request_id_var.set(TEST_REQUEST_ID) - with patch("noteflow.grpc.interceptors.logging.logger") as mock_logger: + with patch("noteflow.grpc.interceptors.logging._logging_ops.logger") as mock_logger: wrapped_handler = await interceptor.intercept_service(continuation, details) typed_handler = cast(_UnaryUnaryHandler, wrapped_handler) @@ -249,7 +253,7 @@ class TestRequestLoggingInterceptor: details = create_handler_call_details(method=TEST_METHOD) continuation = create_mock_continuation(handler) - with patch("noteflow.grpc.interceptors.logging.logger") as mock_logger: + with patch("noteflow.grpc.interceptors.logging._logging_ops.logger") as mock_logger: wrapped_handler = await interceptor.intercept_service(continuation, details) typed_handler = cast(_UnaryUnaryHandler, wrapped_handler) @@ -285,7 +289,7 @@ class TestRequestLoggingInterceptor: details = create_handler_call_details() continuation = create_mock_continuation(handler) - with patch("noteflow.grpc.interceptors.logging.logger") as mock_logger: + with patch("noteflow.grpc.interceptors.logging._logging_ops.logger") as mock_logger: wrapped_handler = await interceptor.intercept_service(continuation, details) typed_handler = cast(_UnaryUnaryHandler, wrapped_handler) diff --git a/tests/grpc/test_meeting_mixin.py b/tests/grpc/test_meeting_mixin.py index 929fd9c..51ec5f9 100644 --- a/tests/grpc/test_meeting_mixin.py +++ b/tests/grpc/test_meeting_mixin.py @@ -8,20 +8,29 @@ Tests cover: - DeleteMeeting: success, not found """ - from __future__ import annotations from typing import cast from unittest.mock import AsyncMock, MagicMock -from uuid import uuid4 +from uuid import UUID, uuid4 import pytest -from noteflow.domain.entities import Meeting +from noteflow.domain.entities import Meeting, Segment +from noteflow.domain.identity import ( + DEFAULT_USER_ID, + DEFAULT_WORKSPACE_ID, + OperationContext, + UserContext, + WorkspaceContext, + WorkspaceRole, +) from noteflow.domain.ports.repositories.identity import ProjectRepository, WorkspaceRepository from noteflow.domain.value_objects import MeetingId, MeetingState -from noteflow.grpc._mixins.meeting import MeetingMixin +from noteflow.grpc.mixins._types import GrpcContext +from noteflow.grpc.mixins.meeting import MeetingMixin from noteflow.grpc.proto import noteflow_pb2 +from noteflow.infrastructure.logging import request_id_var, user_id_var, workspace_id_var from .proto_types import ( CreateMeetingRequestProto, @@ -66,8 +75,8 @@ class MockMeetingRepositoryProvider: self.diarization_jobs = diarization_jobs_repo or AsyncMock() self.supports_diarization_jobs = diarization_jobs_repo is not None self.projects: ProjectRepository = projects_repo or MagicMock(spec=ProjectRepository) - self.workspaces: WorkspaceRepository = ( - workspaces_repo or MagicMock(spec=WorkspaceRepository) + self.workspaces: WorkspaceRepository = workspaces_repo or MagicMock( + spec=WorkspaceRepository ) self.supports_projects = supports_projects self.supports_workspaces = supports_workspaces @@ -112,8 +121,8 @@ class MockMeetingMixinServicerHost(MeetingMixin): self._summaries_repo = summaries_repo or AsyncMock() self.diarization_jobs_repo = diarization_jobs_repo self._projects_repo: ProjectRepository = projects_repo or MagicMock(spec=ProjectRepository) - self._workspaces_repo: WorkspaceRepository = ( - workspaces_repo or MagicMock(spec=WorkspaceRepository) + self._workspaces_repo: WorkspaceRepository = workspaces_repo or MagicMock( + spec=WorkspaceRepository ) # Streaming state required by StopMeeting @@ -143,6 +152,25 @@ class MockMeetingMixinServicerHost(MeetingMixin): """Mock audio writer close.""" self.audio_writers.pop(meeting_id, None) + def get_operation_context(self, context: GrpcContext) -> OperationContext: + """Get operation context from gRPC context variables.""" + request_id = request_id_var.get() + user_id_str = user_id_var.get() + workspace_id_str = workspace_id_var.get() + + user_id = UUID(user_id_str) if user_id_str else DEFAULT_USER_ID + workspace_id = UUID(workspace_id_str) if workspace_id_str else DEFAULT_WORKSPACE_ID + + return OperationContext( + user=UserContext(user_id=user_id, display_name=""), + workspace=WorkspaceContext( + workspace_id=workspace_id, + workspace_name="", + role=WorkspaceRole.OWNER, + ), + request_id=request_id, + ) + # ============================================================================ # Fixtures @@ -213,9 +241,7 @@ class TestCreateMeeting: expected_meeting = Meeting.create(title="Team Standup") meeting_mixin_meetings_repo.create.return_value = expected_meeting - request: CreateMeetingRequestProto = noteflow_pb2.CreateMeetingRequest( - title="Team Standup" - ) + request: CreateMeetingRequestProto = noteflow_pb2.CreateMeetingRequest(title="Team Standup") response: MeetingProto = await meeting_mixin_servicer.CreateMeeting( request, mock_grpc_context ) @@ -322,18 +348,28 @@ class TestStopMeeting: assert response.state == MeetingState.STOPPED.value, "State should be STOPPED" meeting_mixin_meetings_repo.update.assert_called_once() - async def test_stop_is_idempotent_for_stopped_meeting( + @pytest.mark.parametrize( + ("initial_state", "title_suffix"), + [ + pytest.param(MeetingState.STOPPED, "Stopped", id="stopped"), + pytest.param(MeetingState.STOPPING, "Stopping", id="stopping"), + pytest.param(MeetingState.COMPLETED, "Completed", id="completed"), + ], + ) + async def test_stop_is_idempotent( self, + initial_state: MeetingState, + title_suffix: str, meeting_mixin_servicer: MeetingServicerProtocol, meeting_mixin_meetings_repo: AsyncMock, mock_grpc_context: MagicMock, ) -> None: - """StopMeeting returns success for already STOPPED meeting (idempotent).""" + """StopMeeting returns success for already-stopped meetings (idempotent).""" meeting_id = MeetingId(uuid4()) - stopped_meeting = Meeting.create(title="Already Stopped") - stopped_meeting.id = meeting_id - stopped_meeting.state = MeetingState.STOPPED - meeting_mixin_meetings_repo.get.return_value = stopped_meeting + meeting = Meeting.create(title=f"Already {title_suffix}") + meeting.id = meeting_id + meeting.state = initial_state + meeting_mixin_meetings_repo.get.return_value = meeting request: StopMeetingRequestProto = noteflow_pb2.StopMeetingRequest( meeting_id=str(meeting_id) @@ -342,53 +378,7 @@ class TestStopMeeting: request, mock_grpc_context ) - assert response.state == MeetingState.STOPPED.value, "State should remain STOPPED" - meeting_mixin_meetings_repo.update.assert_not_called() - - async def test_stop_is_idempotent_for_stopping_meeting( - self, - meeting_mixin_servicer: MeetingServicerProtocol, - meeting_mixin_meetings_repo: AsyncMock, - mock_grpc_context: MagicMock, - ) -> None: - """StopMeeting returns success for STOPPING meeting (idempotent).""" - meeting_id = MeetingId(uuid4()) - stopping_meeting = Meeting.create(title="Currently Stopping") - stopping_meeting.id = meeting_id - stopping_meeting.state = MeetingState.STOPPING - meeting_mixin_meetings_repo.get.return_value = stopping_meeting - - request: StopMeetingRequestProto = noteflow_pb2.StopMeetingRequest( - meeting_id=str(meeting_id) - ) - response: MeetingProto = await meeting_mixin_servicer.StopMeeting( - request, mock_grpc_context - ) - - assert response.state == MeetingState.STOPPING.value, "State should remain STOPPING" - meeting_mixin_meetings_repo.update.assert_not_called() - - async def test_stop_is_idempotent_for_completed_meeting( - self, - meeting_mixin_servicer: MeetingServicerProtocol, - meeting_mixin_meetings_repo: AsyncMock, - mock_grpc_context: MagicMock, - ) -> None: - """StopMeeting returns success for COMPLETED meeting (idempotent).""" - meeting_id = MeetingId(uuid4()) - completed_meeting = Meeting.create(title="Already Completed") - completed_meeting.id = meeting_id - completed_meeting.state = MeetingState.COMPLETED - meeting_mixin_meetings_repo.get.return_value = completed_meeting - - request: StopMeetingRequestProto = noteflow_pb2.StopMeetingRequest( - meeting_id=str(meeting_id) - ) - response: MeetingProto = await meeting_mixin_servicer.StopMeeting( - request, mock_grpc_context - ) - - assert response.state == MeetingState.COMPLETED.value, "State should remain COMPLETED" + assert response.state == initial_state.value, f"State should remain {initial_state.name}" meeting_mixin_meetings_repo.update.assert_not_called() async def test_stop_meeting_aborts_for_not_found( @@ -465,8 +455,10 @@ class TestStopMeeting: servicer: MeetingServicerProtocol = cast( MeetingServicerProtocol, MockMeetingMixinServicerHost( - meeting_mixin_meetings_repo, meeting_mixin_segments_repo, - meeting_mixin_summaries_repo, webhook_service=mock_webhook_service, + meeting_mixin_meetings_repo, + meeting_mixin_segments_repo, + meeting_mixin_summaries_repo, + webhook_service=mock_webhook_service, ), ) @@ -476,7 +468,9 @@ class TestStopMeeting: recording_meeting.start_recording() meeting_mixin_meetings_repo.get.return_value = recording_meeting - request: StopMeetingRequestProto = noteflow_pb2.StopMeetingRequest(meeting_id=str(meeting_id)) + request: StopMeetingRequestProto = noteflow_pb2.StopMeetingRequest( + meeting_id=str(meeting_id) + ) await servicer.StopMeeting(request, mock_grpc_context) mock_webhook_service.trigger_recording_stopped.assert_called_once() @@ -701,23 +695,19 @@ class TestGetMeeting: mock_grpc_context: MagicMock, ) -> None: """GetMeeting loads segments when include_segments is True.""" - from noteflow.domain.entities import Segment - meeting_id = MeetingId(uuid4()) meeting = Meeting.create(title="Meeting with Segments") meeting.id = meeting_id meeting_mixin_meetings_repo.get.return_value = meeting segments = [ - Segment(segment_id=i, text=text, start_time=float(i), end_time=float(i + 1), meeting_id=meeting_id) - for i, text in enumerate(["Hello world", "Goodbye world"]) + Segment(segment_id=i, text=t, start_time=float(i), end_time=float(i + 1), meeting_id=meeting_id) + for i, t in enumerate(["Hello world", "Goodbye world"]) ] meeting_mixin_segments_repo.get_by_meeting.return_value = segments - request: GetMeetingRequestProto = noteflow_pb2.GetMeetingRequest( - meeting_id=str(meeting_id), include_segments=True, - ) - response: MeetingProto = await meeting_mixin_servicer.GetMeeting(request, mock_grpc_context) + request = noteflow_pb2.GetMeetingRequest(meeting_id=str(meeting_id), include_segments=True) + response = await meeting_mixin_servicer.GetMeeting(request, mock_grpc_context) assert len(response.segments) == 2, "Should include 2 segments" assert response.segments[0].text == "Hello world", "First segment text should match" diff --git a/tests/grpc/test_mixin_helpers.py b/tests/grpc/test_mixin_helpers.py index a70f3a4..3a7f0a2 100644 --- a/tests/grpc/test_mixin_helpers.py +++ b/tests/grpc/test_mixin_helpers.py @@ -13,7 +13,7 @@ import pytest from noteflow.domain.ports.unit_of_work import UnitOfWork from noteflow.domain.value_objects import MeetingId -from noteflow.grpc._mixins.errors import ( +from noteflow.grpc.mixins.errors import ( AbortableContext, get_meeting_or_abort, get_project_or_abort, diff --git a/tests/grpc/test_oauth.py b/tests/grpc/test_oauth.py index 39a9958..63ff352 100644 --- a/tests/grpc/test_oauth.py +++ b/tests/grpc/test_oauth.py @@ -17,7 +17,7 @@ import pytest from noteflow.application.services.calendar import CalendarServiceError from noteflow.domain.entities.integration import IntegrationStatus -from noteflow.grpc._config import ServicesConfig +from noteflow.grpc.config.config import ServicesConfig from noteflow.grpc.proto import noteflow_pb2 from noteflow.grpc.service import NoteFlowServicer diff --git a/tests/grpc/test_observability_mixin.py b/tests/grpc/test_observability_mixin.py index 86da528..e288116 100644 --- a/tests/grpc/test_observability_mixin.py +++ b/tests/grpc/test_observability_mixin.py @@ -12,8 +12,8 @@ from unittest.mock import MagicMock, patch import pytest -from noteflow.grpc._mixins._types import GrpcContext -from noteflow.grpc._mixins.observability import ObservabilityMixin +from noteflow.grpc.mixins._types import GrpcContext +from noteflow.grpc.mixins.observability import ObservabilityMixin from noteflow.grpc.proto import noteflow_pb2 from noteflow.infrastructure.logging.log_buffer import LogBuffer, LogEntry from noteflow.infrastructure.metrics.collector import MetricsCollector, PerformanceMetrics @@ -140,7 +140,7 @@ class TestGetRecentLogs: mock_log_buffer.append(sample_log_entries[3]) with patch( - "noteflow.grpc._mixins.observability.get_log_buffer", + "noteflow.grpc.mixins.observability.get_log_buffer", return_value=mock_log_buffer, ): request = noteflow_pb2.GetRecentLogsRequest() @@ -158,7 +158,7 @@ class TestGetRecentLogs: mock_buffer.get_recent.return_value = [] with patch( - "noteflow.grpc._mixins.observability.get_log_buffer", + "noteflow.grpc.mixins.observability.get_log_buffer", return_value=mock_buffer, ): request = noteflow_pb2.GetRecentLogsRequest() @@ -178,7 +178,7 @@ class TestGetRecentLogs: mock_buffer.get_recent.return_value = [] with patch( - "noteflow.grpc._mixins.observability.get_log_buffer", + "noteflow.grpc.mixins.observability.get_log_buffer", return_value=mock_buffer, ): request = noteflow_pb2.GetRecentLogsRequest(limit=50) @@ -198,7 +198,7 @@ class TestGetRecentLogs: mock_buffer.get_recent.return_value = [] with patch( - "noteflow.grpc._mixins.observability.get_log_buffer", + "noteflow.grpc.mixins.observability.get_log_buffer", return_value=mock_buffer, ): request = noteflow_pb2.GetRecentLogsRequest(limit=2000) @@ -218,7 +218,7 @@ class TestGetRecentLogs: mock_buffer.get_recent.return_value = [] with patch( - "noteflow.grpc._mixins.observability.get_log_buffer", + "noteflow.grpc.mixins.observability.get_log_buffer", return_value=mock_buffer, ): request = noteflow_pb2.GetRecentLogsRequest(level="error") @@ -238,7 +238,7 @@ class TestGetRecentLogs: mock_buffer.get_recent.return_value = [] with patch( - "noteflow.grpc._mixins.observability.get_log_buffer", + "noteflow.grpc.mixins.observability.get_log_buffer", return_value=mock_buffer, ): request = noteflow_pb2.GetRecentLogsRequest(source="api") @@ -258,7 +258,7 @@ class TestGetRecentLogs: mock_buffer.get_recent.return_value = [] with patch( - "noteflow.grpc._mixins.observability.get_log_buffer", + "noteflow.grpc.mixins.observability.get_log_buffer", return_value=mock_buffer, ): request = noteflow_pb2.GetRecentLogsRequest(level="warning", source="sync") @@ -278,7 +278,7 @@ class TestGetRecentLogs: mock_buffer.get_recent.return_value = [] with patch( - "noteflow.grpc._mixins.observability.get_log_buffer", + "noteflow.grpc.mixins.observability.get_log_buffer", return_value=mock_buffer, ): request = noteflow_pb2.GetRecentLogsRequest() @@ -303,7 +303,7 @@ class TestGetRecentLogs: mock_buffer.get_recent.return_value = [entry] with patch( - "noteflow.grpc._mixins.observability.get_log_buffer", + "noteflow.grpc.mixins.observability.get_log_buffer", return_value=mock_buffer, ): request = noteflow_pb2.GetRecentLogsRequest() @@ -336,7 +336,7 @@ class TestGetRecentLogs: mock_buffer.get_recent.return_value = [entry] with patch( - "noteflow.grpc._mixins.observability.get_log_buffer", + "noteflow.grpc.mixins.observability.get_log_buffer", return_value=mock_buffer, ): request = noteflow_pb2.GetRecentLogsRequest() @@ -366,7 +366,7 @@ class TestGetRecentLogs: mock_buffer.get_recent.return_value = [entry] with patch( - "noteflow.grpc._mixins.observability.get_log_buffer", + "noteflow.grpc.mixins.observability.get_log_buffer", return_value=mock_buffer, ): request = noteflow_pb2.GetRecentLogsRequest() @@ -399,7 +399,7 @@ class TestGetRecentLogs: mock_buffer.get_recent.return_value = [entry] with patch( - "noteflow.grpc._mixins.observability.get_log_buffer", + "noteflow.grpc.mixins.observability.get_log_buffer", return_value=mock_buffer, ): request = noteflow_pb2.GetRecentLogsRequest() @@ -433,7 +433,7 @@ class TestGetPerformanceMetrics: mock_collector.get_history.return_value = [] with patch( - "noteflow.grpc._mixins.observability.get_metrics_collector", + "noteflow.grpc.mixins.observability.get_metrics_collector", return_value=mock_collector, ): request = noteflow_pb2.GetPerformanceMetricsRequest() @@ -469,7 +469,7 @@ class TestGetPerformanceMetrics: mock_collector.collect_now.return_value = sample_metrics mock_collector.get_history.return_value = history - with patch("noteflow.grpc._mixins.observability.get_metrics_collector", return_value=mock_collector): + with patch("noteflow.grpc.mixins.observability.get_metrics_collector", return_value=mock_collector): request = noteflow_pb2.GetPerformanceMetricsRequest() response = await observability_servicer.GetPerformanceMetrics(request, mock_grpc_context) @@ -489,7 +489,7 @@ class TestGetPerformanceMetrics: mock_collector.get_history.return_value = [] with patch( - "noteflow.grpc._mixins.observability.get_metrics_collector", + "noteflow.grpc.mixins.observability.get_metrics_collector", return_value=mock_collector, ): request = noteflow_pb2.GetPerformanceMetricsRequest() @@ -509,7 +509,7 @@ class TestGetPerformanceMetrics: mock_collector.get_history.return_value = [] with patch( - "noteflow.grpc._mixins.observability.get_metrics_collector", + "noteflow.grpc.mixins.observability.get_metrics_collector", return_value=mock_collector, ): request = noteflow_pb2.GetPerformanceMetricsRequest(history_limit=30) @@ -529,7 +529,7 @@ class TestGetPerformanceMetrics: mock_collector.get_history.return_value = [] with patch( - "noteflow.grpc._mixins.observability.get_metrics_collector", + "noteflow.grpc.mixins.observability.get_metrics_collector", return_value=mock_collector, ): request = noteflow_pb2.GetPerformanceMetricsRequest() @@ -562,7 +562,7 @@ class TestMetricsProtoConversion: mock_collector.collect_now.return_value = metrics mock_collector.get_history.return_value = [] - with patch("noteflow.grpc._mixins.observability.get_metrics_collector", return_value=mock_collector): + with patch("noteflow.grpc.mixins.observability.get_metrics_collector", return_value=mock_collector): request = noteflow_pb2.GetPerformanceMetricsRequest() response = await observability_servicer.GetPerformanceMetrics(request, mock_grpc_context) @@ -597,7 +597,7 @@ class TestMetricsProtoConversion: mock_collector.get_history.return_value = [] with patch( - "noteflow.grpc._mixins.observability.get_metrics_collector", + "noteflow.grpc.mixins.observability.get_metrics_collector", return_value=mock_collector, ): request = noteflow_pb2.GetPerformanceMetricsRequest() @@ -630,7 +630,7 @@ class TestMetricsProtoConversion: mock_collector.get_history.return_value = [] with patch( - "noteflow.grpc._mixins.observability.get_metrics_collector", + "noteflow.grpc.mixins.observability.get_metrics_collector", return_value=mock_collector, ): request = noteflow_pb2.GetPerformanceMetricsRequest() diff --git a/tests/grpc/test_oidc_mixin.py b/tests/grpc/test_oidc_mixin.py index 6485cac..eae6633 100644 --- a/tests/grpc/test_oidc_mixin.py +++ b/tests/grpc/test_oidc_mixin.py @@ -24,8 +24,8 @@ from noteflow.domain.auth.oidc import ( OidcProviderConfig, OidcProviderPreset, ) -from noteflow.grpc._mixins._types import GrpcContext -from noteflow.grpc._mixins.oidc import OidcMixin +from noteflow.grpc.mixins._types import GrpcContext +from noteflow.grpc.mixins.oidc import OidcMixin from noteflow.grpc.proto import noteflow_pb2 from noteflow.infrastructure.auth.oidc_discovery import OidcDiscoveryError diff --git a/tests/grpc/test_partial_transcription.py b/tests/grpc/test_partial_transcription.py index ecf8db2..7a2272f 100644 --- a/tests/grpc/test_partial_transcription.py +++ b/tests/grpc/test_partial_transcription.py @@ -112,7 +112,7 @@ class TestPartialTranscriptionState: def testinit_streaming_state_createslast_partial_time(self) -> None: """Initialize streaming state should set last partial time to now.""" servicer: ExposedServicer = ExposedServicer() - before = time.time() + before = time.monotonic() servicer.init_streaming_state("meeting-123", next_segment_id=0) @@ -174,7 +174,7 @@ class TestClearPartialBuffer: state = servicer.get_stream_state("meeting-123") assert state is not None state.last_partial_time = 0.0 - before = time.time() + before = time.monotonic() servicer.clear_partial_buffer("meeting-123") @@ -209,7 +209,7 @@ class TestMaybeEmitPartial: # Set last time to now (cadence not reached) state = servicer.get_stream_state("meeting-123") assert state is not None - state.last_partial_time = time.time() + state.last_partial_time = time.monotonic() # Add some audio audio: NDArray[np.float32] = np.ones(DEFAULT_SAMPLE_RATE, dtype=np.float32) * 0.1 # 1 second of audio state.partial_buffer.append(audio) @@ -227,7 +227,7 @@ class TestMaybeEmitPartial: # Set last time to past (cadence reached) state = servicer.get_stream_state("meeting-123") assert state is not None - state.last_partial_time = time.time() - 10.0 + state.last_partial_time = time.monotonic() - 10.0 result: noteflow_pb2.TranscriptUpdate | None = await servicer.maybe_emit_partial("meeting-123") @@ -241,7 +241,7 @@ class TestMaybeEmitPartial: servicer.init_streaming_state("meeting-123", next_segment_id=0) state = servicer.get_stream_state("meeting-123") assert state is not None - state.last_partial_time = time.time() - 10.0 + state.last_partial_time = time.monotonic() - 10.0 # Add only 0.1 seconds of audio (minimum is 0.5s) audio: NDArray[np.float32] = np.ones(1600, dtype=np.float32) * 0.1 # 0.1 second state.partial_buffer.append(audio) @@ -258,7 +258,7 @@ class TestMaybeEmitPartial: servicer.init_streaming_state("meeting-123", next_segment_id=0) state = servicer.get_stream_state("meeting-123") assert state is not None - state.last_partial_time = time.time() - 10.0 + state.last_partial_time = time.monotonic() - 10.0 # Add 1 second of audio (above minimum of 0.5s) audio: NDArray[np.float32] = np.ones(DEFAULT_SAMPLE_RATE, dtype=np.float32) * 0.1 state.partial_buffer.append(audio) @@ -278,7 +278,7 @@ class TestMaybeEmitPartial: servicer.init_streaming_state("meeting-123", next_segment_id=0) state = servicer.get_stream_state("meeting-123") assert state is not None - state.last_partial_time = time.time() - 10.0 + state.last_partial_time = time.monotonic() - 10.0 state.last_partial_text = "Same text" # Same as transcription audio: NDArray[np.float32] = np.ones(DEFAULT_SAMPLE_RATE, dtype=np.float32) * 0.1 state.partial_buffer.append(audio) @@ -295,10 +295,10 @@ class TestMaybeEmitPartial: servicer.init_streaming_state("meeting-123", next_segment_id=0) state = servicer.get_stream_state("meeting-123") assert state is not None - state.last_partial_time = time.time() - 10.0 + state.last_partial_time = time.monotonic() - 10.0 audio: NDArray[np.float32] = np.ones(DEFAULT_SAMPLE_RATE, dtype=np.float32) * 0.1 state.partial_buffer.append(audio) - before = time.time() + before = time.monotonic() await servicer.maybe_emit_partial("meeting-123") diff --git a/tests/grpc/test_preferences_mixin.py b/tests/grpc/test_preferences_mixin.py index a4d5a67..b587645 100644 --- a/tests/grpc/test_preferences_mixin.py +++ b/tests/grpc/test_preferences_mixin.py @@ -16,8 +16,8 @@ from unittest.mock import AsyncMock, MagicMock import pytest -from noteflow.grpc._mixins._types import GrpcContext -from noteflow.grpc._mixins.preferences import PreferencesMixin, compute_etag +from noteflow.grpc.mixins._types import GrpcContext +from noteflow.grpc.mixins.preferences import PreferencesMixin, compute_etag from noteflow.grpc.proto import noteflow_pb2 from noteflow.infrastructure.persistence.repositories.preferences_repo import ( PreferenceWithMetadata, diff --git a/tests/grpc/test_project_mixin.py b/tests/grpc/test_project_mixin.py index 16fc803..bd4122c 100644 --- a/tests/grpc/test_project_mixin.py +++ b/tests/grpc/test_project_mixin.py @@ -38,7 +38,7 @@ from noteflow.domain.ports.repositories.identity import ( WorkspaceRepository, ) from noteflow.domain.value_objects import ExportFormat -from noteflow.grpc._mixins.project import ProjectMembershipMixin, ProjectMixin +from noteflow.grpc.mixins.project import ProjectMembershipMixin, ProjectMixin from noteflow.grpc.proto import noteflow_pb2 # ============================================================================ diff --git a/tests/grpc/test_proto_compilation.py b/tests/grpc/test_proto_compilation.py index 898b058..1fd1d0b 100644 --- a/tests/grpc/test_proto_compilation.py +++ b/tests/grpc/test_proto_compilation.py @@ -84,7 +84,7 @@ class TestSprint0RPCs: """Verify Sprint 0 RPC stubs exist.""" def test_servicer_has_extract_entities(self) -> None: - """NoteFlowServicer has ExtractEntities method.""" + """NoteFlowServicer has ExtractEntities method on stub.""" from noteflow.grpc.proto import noteflow_pb2_grpc servicer = noteflow_pb2_grpc.NoteFlowServiceServicer diff --git a/tests/grpc/test_server_auto_enable.py b/tests/grpc/test_server_auto_enable.py index fe0875a..6832172 100644 --- a/tests/grpc/test_server_auto_enable.py +++ b/tests/grpc/test_server_auto_enable.py @@ -18,7 +18,7 @@ from noteflow.application.services.summarization import ( SummarizationServiceSettings, ) from noteflow.domain.entities.integration import Integration, IntegrationStatus, IntegrationType -from noteflow.grpc._startup import ( +from noteflow.grpc.startup.startup import ( auto_enable_cloud_llm, check_calendar_needed_from_db, ) @@ -234,7 +234,7 @@ class TestAutoEnableCloudLlm: service = _create_mocksummarization_service() with patch( - "noteflow.grpc._startup.CloudSummarizer" + "noteflow.grpc.startup.startup.CloudSummarizer" ) as mock_cloud_summarizer_class: mock_summarizer = MagicMock() mock_cloud_summarizer_class.return_value = mock_summarizer @@ -246,6 +246,7 @@ class TestAutoEnableCloudLlm: backend=CloudBackend.OPENAI, api_key="sk-test-openai-key", model="gpt-4o", + base_url=None, ) service.register_provider.assert_called_once_with( SummarizationMode.CLOUD, mock_summarizer @@ -268,7 +269,7 @@ class TestAutoEnableCloudLlm: service = _create_mocksummarization_service() with patch( - "noteflow.grpc._startup.CloudSummarizer" + "noteflow.grpc.startup.startup.CloudSummarizer" ) as mock_cloud_summarizer_class: mock_summarizer = MagicMock() mock_cloud_summarizer_class.return_value = mock_summarizer @@ -280,6 +281,7 @@ class TestAutoEnableCloudLlm: backend=CloudBackend.ANTHROPIC, api_key="sk-ant-test-key", model="claude-3-5-sonnet-20241022", + base_url=None, ) service.register_provider.assert_called_once_with( SummarizationMode.CLOUD, mock_summarizer @@ -302,7 +304,7 @@ class TestAutoEnableCloudLlm: service = _create_mocksummarization_service() with patch( - "noteflow.grpc._startup.CloudSummarizer" + "noteflow.grpc.startup.startup.CloudSummarizer" ) as mock_cloud_summarizer_class: await auto_enable_cloud_llm(uow, service) @@ -310,6 +312,7 @@ class TestAutoEnableCloudLlm: backend=CloudBackend.OPENAI, api_key="sk-test-key", model=None, + base_url=None, ) @pytest.mark.asyncio @@ -328,7 +331,7 @@ class TestAutoEnableCloudLlm: service = _create_mocksummarization_service() with patch( - "noteflow.grpc._startup.CloudSummarizer" + "noteflow.grpc.startup.startup.CloudSummarizer" ) as mock_cloud_summarizer_class: await auto_enable_cloud_llm(uow, service) @@ -337,6 +340,7 @@ class TestAutoEnableCloudLlm: backend=CloudBackend.ANTHROPIC, api_key="sk-ant-key", model=None, + base_url=None, ) @@ -556,7 +560,7 @@ class TestAutoEnableEdgeCases: service = _create_mocksummarization_service() with patch( - "noteflow.grpc._startup.CloudSummarizer" + "noteflow.grpc.startup.startup.CloudSummarizer" ) as mock_cloud_summarizer_class: mock_summarizer = MagicMock() mock_cloud_summarizer_class.return_value = mock_summarizer @@ -569,6 +573,7 @@ class TestAutoEnableEdgeCases: backend=CloudBackend.OPENAI, api_key="sk-test-key", model=["gpt-4", "gpt-3.5"], + base_url=None, ) @pytest.mark.asyncio @@ -586,7 +591,7 @@ class TestAutoEnableEdgeCases: service = _create_mocksummarization_service() with patch( - "noteflow.grpc._startup.CloudSummarizer" + "noteflow.grpc.startup.startup.CloudSummarizer" ) as mock_cloud_summarizer_class: mock_cloud_summarizer_class.side_effect = ValueError("Invalid API key format") @@ -616,7 +621,7 @@ class TestAutoEnableWithRealSummarizationService: service = SummarizationService(settings=SummarizationServiceSettings()) with patch( - "noteflow.grpc._startup.CloudSummarizer" + "noteflow.grpc.startup.startup.CloudSummarizer" ) as mock_cloud_summarizer_class: mock_summarizer = MagicMock() mock_summarizer.is_available = True @@ -644,7 +649,7 @@ class TestAutoEnableWithRealSummarizationService: service = SummarizationService(settings=SummarizationServiceSettings()) with patch( - "noteflow.grpc._startup.CloudSummarizer" + "noteflow.grpc.startup.startup.CloudSummarizer" ) as mock_cloud_summarizer_class: mock_summarizer = MagicMock() mock_summarizer.is_available = True diff --git a/tests/grpc/test_server_health.py b/tests/grpc/test_server_health.py new file mode 100644 index 0000000..928a16e --- /dev/null +++ b/tests/grpc/test_server_health.py @@ -0,0 +1,383 @@ +"""Tests for ServerHealthState class. + +Tests the server bootstrap health tracking functionality including +state management, error recording, and health status reporting. + +Sprint GAP-003: Error Handling Mismatches +""" + +from __future__ import annotations + +import pytest + +from noteflow.grpc.server.health import ( + ServerHealthState, + get_server_health, + reset_server_health, +) + +# Test constants +JOB_RECOVERY_ERROR_MSG = "Failed to recover jobs from database" +"""Error message for job recovery failures.""" + +CONSENT_LOADING_ERROR_MSG = "Cloud consent cache corrupted" +"""Error message for consent loading failures.""" + +ASR_CONFIG_ERROR_MSG = "Invalid ASR model path" +"""Error message for ASR config loading failures.""" + + +@pytest.fixture +def health_state() -> ServerHealthState: + """Create fresh ServerHealthState instance for testing. + + Returns: + New ServerHealthState instance with default healthy state. + """ + return ServerHealthState() + + +@pytest.fixture(autouse=True) +def reset_global_health() -> None: + """Reset global health singleton before each test. + + Ensures test isolation by clearing any state from previous tests. + """ + reset_server_health() + + +class TestDefaultState: + """Tests for ServerHealthState default initialization.""" + + def test_default_state_is_healthy(self, health_state: ServerHealthState) -> None: + """Verify new instance starts in healthy state.""" + assert health_state.is_healthy is True, ( + "Default state should be healthy" + ) + + def test_default_job_recovery_success( + self, health_state: ServerHealthState + ) -> None: + """Verify job_recovery_success defaults to True.""" + assert health_state.job_recovery_success is True, ( + "Job recovery should default to success" + ) + + def test_default_consent_loading_success( + self, health_state: ServerHealthState + ) -> None: + """Verify consent_loading_success defaults to True.""" + assert health_state.consent_loading_success is True, ( + "Consent loading should default to success" + ) + + def test_default_asr_config_loading_success( + self, health_state: ServerHealthState + ) -> None: + """Verify asr_config_loading_success defaults to True.""" + assert health_state.asr_config_loading_success is True, ( + "ASR config loading should default to success" + ) + + def test_default_error_messages_empty( + self, health_state: ServerHealthState + ) -> None: + """Verify all error messages default to empty strings.""" + assert health_state.job_recovery_error == "", ( + "Job recovery error should default to empty" + ) + assert health_state.consent_loading_error == "", ( + "Consent loading error should default to empty" + ) + assert health_state.asr_config_loading_error == "", ( + "ASR config loading error should default to empty" + ) + + +class TestJobRecoveryTracking: + """Tests for job recovery state tracking.""" + + def test_mark_job_recovery_failed_sets_flag( + self, health_state: ServerHealthState + ) -> None: + """Verify setting job_recovery_success to False marks failure.""" + health_state.job_recovery_success = False + health_state.job_recovery_error = JOB_RECOVERY_ERROR_MSG + + assert health_state.job_recovery_success is False, ( + "Job recovery should be marked as failed" + ) + + def test_mark_job_recovery_failed_stores_error( + self, health_state: ServerHealthState + ) -> None: + """Verify job recovery error message is stored.""" + health_state.job_recovery_success = False + health_state.job_recovery_error = JOB_RECOVERY_ERROR_MSG + + assert health_state.job_recovery_error == JOB_RECOVERY_ERROR_MSG, ( + "Job recovery error message should be stored" + ) + + def test_job_recovery_failure_makes_unhealthy( + self, health_state: ServerHealthState + ) -> None: + """Verify job recovery failure makes overall state unhealthy.""" + health_state.job_recovery_success = False + + assert health_state.is_healthy is False, ( + "State should be unhealthy after job recovery failure" + ) + + +class TestConsentLoadingTracking: + """Tests for consent loading state tracking.""" + + def test_mark_consent_loading_failed_sets_flag( + self, health_state: ServerHealthState + ) -> None: + """Verify setting consent_loading_success to False marks failure.""" + health_state.consent_loading_success = False + health_state.consent_loading_error = CONSENT_LOADING_ERROR_MSG + + assert health_state.consent_loading_success is False, ( + "Consent loading should be marked as failed" + ) + + def test_mark_consent_loading_failed_stores_error( + self, health_state: ServerHealthState + ) -> None: + """Verify consent loading error message is stored.""" + health_state.consent_loading_success = False + health_state.consent_loading_error = CONSENT_LOADING_ERROR_MSG + + assert health_state.consent_loading_error == CONSENT_LOADING_ERROR_MSG, ( + "Consent loading error message should be stored" + ) + + def test_consent_loading_failure_makes_unhealthy( + self, health_state: ServerHealthState + ) -> None: + """Verify consent loading failure makes overall state unhealthy.""" + health_state.consent_loading_success = False + + assert health_state.is_healthy is False, ( + "State should be unhealthy after consent loading failure" + ) + + +class TestAsrConfigTracking: + """Tests for ASR config loading state tracking.""" + + def test_mark_asr_config_failed_sets_flag( + self, health_state: ServerHealthState + ) -> None: + """Verify setting asr_config_loading_success to False marks failure.""" + health_state.asr_config_loading_success = False + health_state.asr_config_loading_error = ASR_CONFIG_ERROR_MSG + + assert health_state.asr_config_loading_success is False, ( + "ASR config loading should be marked as failed" + ) + + def test_mark_asr_config_failed_stores_error( + self, health_state: ServerHealthState + ) -> None: + """Verify ASR config error message is stored.""" + health_state.asr_config_loading_success = False + health_state.asr_config_loading_error = ASR_CONFIG_ERROR_MSG + + assert health_state.asr_config_loading_error == ASR_CONFIG_ERROR_MSG, ( + "ASR config error message should be stored" + ) + + def test_asr_config_failure_makes_unhealthy( + self, health_state: ServerHealthState + ) -> None: + """Verify ASR config failure makes overall state unhealthy.""" + health_state.asr_config_loading_success = False + + assert health_state.is_healthy is False, ( + "State should be unhealthy after ASR config loading failure" + ) + + +class TestIsHealthyProperty: + """Tests for is_healthy property logic.""" + + def test_healthy_when_all_succeed(self, health_state: ServerHealthState) -> None: + """Verify is_healthy is True when all operations succeed.""" + # Default state is all success + assert health_state.is_healthy is True, ( + "Should be healthy when all operations succeed" + ) + + @pytest.mark.parametrize( + ("job_recovery", "consent_loading", "asr_config", "expected_healthy"), + [ + pytest.param(False, True, True, False, id="job_recovery_failed"), + pytest.param(True, False, True, False, id="consent_loading_failed"), + pytest.param(True, True, False, False, id="asr_config_failed"), + pytest.param(False, False, True, False, id="two_failures"), + pytest.param(False, False, False, False, id="all_failed"), + pytest.param(True, True, True, True, id="all_succeed"), + ], + ) + def test_health_combinations( + self, + health_state: ServerHealthState, + job_recovery: bool, + consent_loading: bool, + asr_config: bool, + expected_healthy: bool, + ) -> None: + """Verify is_healthy reflects correct combination of operation results.""" + health_state.job_recovery_success = job_recovery + health_state.consent_loading_success = consent_loading + health_state.asr_config_loading_success = asr_config + + assert health_state.is_healthy is expected_healthy, ( + f"Expected is_healthy={expected_healthy} for job_recovery={job_recovery}, " + f"consent_loading={consent_loading}, asr_config={asr_config}" + ) + + +class TestGetErrorSummary: + """Tests for get_error_summary method.""" + + def test_returns_empty_string_when_healthy( + self, health_state: ServerHealthState + ) -> None: + """Verify get_error_summary returns empty string when healthy.""" + summary = health_state.get_error_summary() + + assert summary == "", "Error summary should be empty when healthy" + + def test_includes_job_recovery_error( + self, health_state: ServerHealthState + ) -> None: + """Verify error summary includes job recovery failure.""" + health_state.job_recovery_success = False + health_state.job_recovery_error = JOB_RECOVERY_ERROR_MSG + + summary = health_state.get_error_summary() + + assert "job_recovery" in summary, "Summary should mention job_recovery" + assert JOB_RECOVERY_ERROR_MSG in summary, ( + "Summary should include error message" + ) + + def test_includes_consent_loading_error( + self, health_state: ServerHealthState + ) -> None: + """Verify error summary includes consent loading failure.""" + health_state.consent_loading_success = False + health_state.consent_loading_error = CONSENT_LOADING_ERROR_MSG + + summary = health_state.get_error_summary() + + assert "consent_loading" in summary, "Summary should mention consent_loading" + assert CONSENT_LOADING_ERROR_MSG in summary, ( + "Summary should include error message" + ) + + def test_includes_asr_config_error( + self, health_state: ServerHealthState + ) -> None: + """Verify error summary includes ASR config failure.""" + health_state.asr_config_loading_success = False + health_state.asr_config_loading_error = ASR_CONFIG_ERROR_MSG + + summary = health_state.get_error_summary() + + assert "asr_config_loading" in summary, ( + "Summary should mention asr_config_loading" + ) + assert ASR_CONFIG_ERROR_MSG in summary, ( + "Summary should include error message" + ) + + def test_combines_multiple_errors_with_semicolon( + self, health_state: ServerHealthState + ) -> None: + """Verify multiple errors are combined with semicolon separator.""" + health_state.job_recovery_success = False + health_state.job_recovery_error = JOB_RECOVERY_ERROR_MSG + health_state.consent_loading_success = False + health_state.consent_loading_error = CONSENT_LOADING_ERROR_MSG + + summary = health_state.get_error_summary() + + assert "; " in summary, "Multiple errors should be separated by semicolon" + assert "job_recovery" in summary, "Summary should include job_recovery" + assert "consent_loading" in summary, "Summary should include consent_loading" + + def test_error_summary_order(self, health_state: ServerHealthState) -> None: + """Verify errors appear in consistent order: job_recovery, consent, asr.""" + health_state.job_recovery_success = False + health_state.job_recovery_error = "error1" + health_state.consent_loading_success = False + health_state.consent_loading_error = "error2" + health_state.asr_config_loading_success = False + health_state.asr_config_loading_error = "error3" + + summary = health_state.get_error_summary() + + job_idx = summary.find("job_recovery") + consent_idx = summary.find("consent_loading") + asr_idx = summary.find("asr_config_loading") + + assert job_idx < consent_idx < asr_idx, ( + "Errors should appear in order: job_recovery, consent_loading, asr_config_loading" + ) + + +class TestGlobalSingleton: + """Tests for global singleton functions.""" + + def test_get_server_health_returns_singleton(self) -> None: + """Verify get_server_health returns same instance.""" + health1 = get_server_health() + health2 = get_server_health() + + assert health1 is health2, "Should return same singleton instance" + + def test_health_singleton_persists_state(self) -> None: + """Verify state persists across get_server_health calls.""" + health = get_server_health() + health.job_recovery_success = False + health.job_recovery_error = JOB_RECOVERY_ERROR_MSG + + # Get again and verify state + health_again = get_server_health() + + assert health_again.job_recovery_success is False, ( + "State should persist across singleton access" + ) + assert health_again.job_recovery_error == JOB_RECOVERY_ERROR_MSG, ( + "Error message should persist across singleton access" + ) + + def test_reset_server_health_clears_singleton(self) -> None: + """Verify reset_server_health clears the singleton.""" + health = get_server_health() + health.job_recovery_success = False + + reset_server_health() + + # New singleton should be fresh + new_health = get_server_health() + + assert new_health.is_healthy is True, ( + "New singleton should be healthy after reset" + ) + + def test_health_reset_creates_new_instance(self) -> None: + """Verify reset_server_health creates a new instance.""" + health_before = get_server_health() + reset_server_health() + health_after = get_server_health() + + assert health_before is not health_after, ( + "Reset should create a new instance" + ) diff --git a/tests/grpc/test_stream_lifecycle.py b/tests/grpc/test_stream_lifecycle.py index 05b194c..a05899b 100644 --- a/tests/grpc/test_stream_lifecycle.py +++ b/tests/grpc/test_stream_lifecycle.py @@ -676,8 +676,7 @@ class TestShutdownEdgeCases: memory_servicer.cleanup_streaming_state(meeting_id) memory_servicer.active_streams.discard(meeting_id) - @pytest.mark.asyncio - def _setup_tracked_task( + async def _setup_tracked_task( self, servicer: NoteFlowServicer, operation_order: list[str], @@ -694,7 +693,7 @@ class TestShutdownEdgeCases: raise task = asyncio.create_task(tracked_task()) - asyncio.get_event_loop().run_until_complete(task_started.wait()) + await task_started.wait() servicer.diarization_tasks["test-job-order-001"] = task return task @@ -1041,7 +1040,7 @@ class TestImprovedCleanupGuarantees: self, memory_servicer: NoteFlowServicer ) -> None: """Verify cleanup removes meeting from active streams.""" - from noteflow.grpc._mixins.streaming._cleanup import cleanup_stream_resources + from noteflow.grpc.mixins.streaming._cleanup import cleanup_stream_resources meeting_id = "test-partial-init-001" memory_servicer.active_streams.add(meeting_id) @@ -1057,7 +1056,7 @@ class TestImprovedCleanupGuarantees: self, memory_servicer: NoteFlowServicer ) -> None: """Verify first cleanup call works correctly.""" - from noteflow.grpc._mixins.streaming._cleanup import cleanup_stream_resources + from noteflow.grpc.mixins.streaming._cleanup import cleanup_stream_resources meeting_id = "test-idempotent-001" memory_servicer.init_streaming_state(meeting_id, next_segment_id=0) @@ -1074,7 +1073,7 @@ class TestImprovedCleanupGuarantees: self, memory_servicer: NoteFlowServicer ) -> None: """Verify second cleanup call does not raise.""" - from noteflow.grpc._mixins.streaming._cleanup import cleanup_stream_resources + from noteflow.grpc.mixins.streaming._cleanup import cleanup_stream_resources meeting_id = "test-idempotent-002" memory_servicer.init_streaming_state(meeting_id, next_segment_id=0) @@ -1092,7 +1091,7 @@ class TestImprovedCleanupGuarantees: self, memory_servicer: NoteFlowServicer ) -> None: """Verify cleanup handles meeting that was never actually initialized.""" - from noteflow.grpc._mixins.streaming._cleanup import cleanup_stream_resources + from noteflow.grpc.mixins.streaming._cleanup import cleanup_stream_resources meeting_id = "test-never-init-001" @@ -1107,7 +1106,7 @@ class TestImprovedCleanupGuarantees: self, memory_servicer: NoteFlowServicer ) -> None: """Verify cleanup works when only active_streams was set.""" - from noteflow.grpc._mixins.streaming._cleanup import cleanup_stream_resources + from noteflow.grpc.mixins.streaming._cleanup import cleanup_stream_resources meeting_id = "test-exception-init-001" memory_servicer.active_streams.add(meeting_id) diff --git a/tests/grpc/test_streaming_metrics.py b/tests/grpc/test_streaming_metrics.py new file mode 100644 index 0000000..61de32d --- /dev/null +++ b/tests/grpc/test_streaming_metrics.py @@ -0,0 +1,295 @@ +"""Tests for StreamingErrorMetrics class. + +Tests the streaming error metrics tracking functionality including +counter increments, stats retrieval, and thread safety. + +Sprint GAP-003: Error Handling Mismatches +""" + +from __future__ import annotations + +import threading +from concurrent.futures import ThreadPoolExecutor + +import pytest + +from noteflow.grpc.mixins._metrics import ( + StreamingErrorMetrics, + get_streaming_metrics, + reset_streaming_metrics, +) + +# Test constants +AUDIO_DECODE_ERROR = "audio_decode_failed" +"""Error type for audio decoding failures.""" + +VAD_ERROR = "vad_processing_failed" +"""Error type for VAD processing failures.""" + +DIARIZATION_ERROR = "diarization_failed" +"""Error type for diarization failures.""" + +MEETING_ID_1 = "meeting-001" +"""Test meeting ID for primary meeting.""" + +MEETING_ID_2 = "meeting-002" +"""Test meeting ID for secondary meeting.""" + +CONCURRENT_THREAD_COUNT = 10 +"""Number of threads for thread-safety tests.""" + +ITERATIONS_PER_THREAD = 100 +"""Iterations per thread for thread-safety tests.""" + + +def _run_concurrent_record_and_read(metrics: StreamingErrorMetrics) -> None: + """Run concurrent record and read operations for thread safety testing.""" + errors_recorded = threading.Event() + + def record_loop() -> None: + for _ in range(ITERATIONS_PER_THREAD): + metrics.record_error(AUDIO_DECODE_ERROR, MEETING_ID_1) + errors_recorded.set() + + def read_loop() -> None: + while not errors_recorded.is_set(): + counts = metrics.get_counts() + _ = counts.get(AUDIO_DECODE_ERROR, 0) + + record_thread = threading.Thread(target=record_loop) + read_thread = threading.Thread(target=read_loop) + record_thread.start() + read_thread.start() + record_thread.join() + read_thread.join() + + +@pytest.fixture +def metrics() -> StreamingErrorMetrics: + """Create fresh StreamingErrorMetrics instance for testing. + + Returns: + New StreamingErrorMetrics instance with no recorded errors. + """ + return StreamingErrorMetrics() + + +@pytest.fixture(autouse=True) +def reset_global_metrics() -> None: + """Reset global metrics singleton before each test. + + Ensures test isolation by clearing any state from previous tests. + """ + reset_streaming_metrics() + + +class TestRecordError: + """Tests for StreamingErrorMetrics.record_error method.""" + + def test_increments_counter_for_single_error_type( + self, metrics: StreamingErrorMetrics + ) -> None: + """Verify recording an error increments the counter for that type.""" + metrics.record_error(AUDIO_DECODE_ERROR, MEETING_ID_1) + + counts = metrics.get_counts() + assert counts[AUDIO_DECODE_ERROR] == 1, ( + "Counter should be 1 after single error recording" + ) + + def test_increments_counter_multiple_times_same_type( + self, metrics: StreamingErrorMetrics + ) -> None: + """Verify repeated recordings increment the same counter.""" + increment_count = 3 + metrics.record_error(AUDIO_DECODE_ERROR, MEETING_ID_1) + metrics.record_error(AUDIO_DECODE_ERROR, MEETING_ID_1) + metrics.record_error(AUDIO_DECODE_ERROR, MEETING_ID_2) + + counts = metrics.get_counts() + assert counts[AUDIO_DECODE_ERROR] == increment_count, ( + f"Counter should be {increment_count} after three recordings" + ) + + def test_tracks_different_error_types_independently( + self, metrics: StreamingErrorMetrics + ) -> None: + """Verify different error types maintain separate counters.""" + metrics.record_error(AUDIO_DECODE_ERROR, MEETING_ID_1) + metrics.record_error(VAD_ERROR, MEETING_ID_1) + metrics.record_error(VAD_ERROR, MEETING_ID_2) + + counts = metrics.get_counts() + assert counts[AUDIO_DECODE_ERROR] == 1, "Audio decode counter should be 1" + assert counts[VAD_ERROR] == 2, "VAD error counter should be 2" + + @pytest.mark.parametrize( + ("error_type", "meeting_id"), + [ + pytest.param(AUDIO_DECODE_ERROR, MEETING_ID_1, id="audio_decode_meeting1"), + pytest.param(VAD_ERROR, MEETING_ID_2, id="vad_error_meeting2"), + pytest.param(DIARIZATION_ERROR, MEETING_ID_1, id="diarization_meeting1"), + ], + ) + def test_records_various_error_types( + self, metrics: StreamingErrorMetrics, error_type: str, meeting_id: str + ) -> None: + """Verify various error types can be recorded successfully.""" + metrics.record_error(error_type, meeting_id) + + counts = metrics.get_counts() + assert error_type in counts, f"Error type '{error_type}' should be in counts" + assert counts[error_type] == 1, f"Counter for '{error_type}' should be 1" + + +class TestGetCounts: + """Tests for StreamingErrorMetrics.get_counts method.""" + + def test_returns_empty_dict_when_no_errors( + self, metrics: StreamingErrorMetrics + ) -> None: + """Verify get_counts returns empty dict with no recorded errors.""" + counts = metrics.get_counts() + + assert counts == {}, "Counts should be empty dict when no errors recorded" + + def test_returns_copy_of_internal_state( + self, metrics: StreamingErrorMetrics + ) -> None: + """Verify get_counts returns a copy, not a reference.""" + metrics.record_error(AUDIO_DECODE_ERROR, MEETING_ID_1) + counts = metrics.get_counts() + + # Modify returned dict + counts[AUDIO_DECODE_ERROR] = 999 + + # Original should be unchanged + assert metrics.get_counts()[AUDIO_DECODE_ERROR] == 1, ( + "Internal state should not be modified by external changes" + ) + + def test_returns_all_recorded_error_types( + self, metrics: StreamingErrorMetrics + ) -> None: + """Verify get_counts includes all recorded error types.""" + metrics.record_error(AUDIO_DECODE_ERROR, MEETING_ID_1) + metrics.record_error(VAD_ERROR, MEETING_ID_1) + metrics.record_error(DIARIZATION_ERROR, MEETING_ID_1) + + counts = metrics.get_counts() + expected_types = {AUDIO_DECODE_ERROR, VAD_ERROR, DIARIZATION_ERROR} + + assert set(counts.keys()) == expected_types, ( + f"Should return all error types: expected {expected_types}, got {set(counts.keys())}" + ) + + +class TestReset: + """Tests for StreamingErrorMetrics.reset method.""" + + def test_clears_all_error_counts(self, metrics: StreamingErrorMetrics) -> None: + """Verify reset clears all recorded errors.""" + metrics.record_error(AUDIO_DECODE_ERROR, MEETING_ID_1) + metrics.record_error(VAD_ERROR, MEETING_ID_2) + + metrics.reset() + + counts = metrics.get_counts() + assert counts == {}, "Counts should be empty after reset" + + def test_allows_new_recordings_after_reset( + self, metrics: StreamingErrorMetrics + ) -> None: + """Verify new errors can be recorded after reset.""" + metrics.record_error(AUDIO_DECODE_ERROR, MEETING_ID_1) + metrics.reset() + metrics.record_error(VAD_ERROR, MEETING_ID_1) + + counts = metrics.get_counts() + assert AUDIO_DECODE_ERROR not in counts, ( + "Old error type should not be present after reset" + ) + assert counts[VAD_ERROR] == 1, "New error should be recorded after reset" + + +class TestThreadSafety: + """Tests for StreamingErrorMetrics thread safety.""" + + def test_concurrent_record_error_preserves_count( + self, metrics: StreamingErrorMetrics + ) -> None: + """Verify concurrent record_error calls produce correct total count.""" + expected_total = CONCURRENT_THREAD_COUNT * ITERATIONS_PER_THREAD + + def record_errors() -> None: + for _ in range(ITERATIONS_PER_THREAD): + metrics.record_error(AUDIO_DECODE_ERROR, MEETING_ID_1) + + with ThreadPoolExecutor(max_workers=CONCURRENT_THREAD_COUNT) as executor: + futures = [executor.submit(record_errors) for _ in range(CONCURRENT_THREAD_COUNT)] + for future in futures: + future.result() + + counts = metrics.get_counts() + assert counts[AUDIO_DECODE_ERROR] == expected_total, ( + f"Expected {expected_total} errors after concurrent recording, " + f"got {counts[AUDIO_DECODE_ERROR]}" + ) + + def test_concurrent_mixed_operations( + self, metrics: StreamingErrorMetrics + ) -> None: + """Verify concurrent record and get_counts operations are safe.""" + _run_concurrent_record_and_read(metrics) + counts = metrics.get_counts() + + assert counts[AUDIO_DECODE_ERROR] == ITERATIONS_PER_THREAD, ( + f"Expected {ITERATIONS_PER_THREAD} errors after concurrent operations" + ) + + +class TestGlobalSingleton: + """Tests for global singleton functions.""" + + def test_get_streaming_metrics_returns_singleton(self) -> None: + """Verify get_streaming_metrics returns same instance.""" + metrics1 = get_streaming_metrics() + metrics2 = get_streaming_metrics() + + assert metrics1 is metrics2, "Should return same singleton instance" + + def test_metrics_singleton_persists_state(self) -> None: + """Verify state persists across get_streaming_metrics calls.""" + metrics = get_streaming_metrics() + metrics.record_error(AUDIO_DECODE_ERROR, MEETING_ID_1) + + # Get again and verify state + metrics_again = get_streaming_metrics() + counts = metrics_again.get_counts() + + assert counts[AUDIO_DECODE_ERROR] == 1, ( + "State should persist across singleton access" + ) + + def test_reset_streaming_metrics_clears_singleton(self) -> None: + """Verify reset_streaming_metrics clears the singleton.""" + metrics = get_streaming_metrics() + metrics.record_error(AUDIO_DECODE_ERROR, MEETING_ID_1) + + reset_streaming_metrics() + + # New singleton should be fresh + new_metrics = get_streaming_metrics() + counts = new_metrics.get_counts() + + assert counts == {}, "New singleton should have no errors after reset" + + def test_metrics_reset_creates_new_instance(self) -> None: + """Verify reset_streaming_metrics creates a new instance.""" + metrics_before = get_streaming_metrics() + reset_streaming_metrics() + metrics_after = get_streaming_metrics() + + assert metrics_before is not metrics_after, ( + "Reset should create a new instance" + ) diff --git a/tests/grpc/test_sync_orchestration.py b/tests/grpc/test_sync_orchestration.py index 4652b46..6178446 100644 --- a/tests/grpc/test_sync_orchestration.py +++ b/tests/grpc/test_sync_orchestration.py @@ -25,7 +25,7 @@ from noteflow.domain.entities.integration import ( IntegrationType, SyncRun, ) -from noteflow.grpc._config import ServicesConfig +from noteflow.grpc.config.config import ServicesConfig from noteflow.grpc.meeting_store import MeetingStore from noteflow.grpc.proto import noteflow_pb2 from noteflow.grpc.service import NoteFlowServicer diff --git a/tests/grpc/test_task_callbacks.py b/tests/grpc/test_task_callbacks.py new file mode 100644 index 0000000..c30159b --- /dev/null +++ b/tests/grpc/test_task_callbacks.py @@ -0,0 +1,403 @@ +"""Tests for job done callback functionality. + +Tests the create_job_done_callback factory for handling background job +task completion including success, failure, and cancellation scenarios. + +Sprint GAP-003: Error Handling Mismatches +""" + +from __future__ import annotations + +import asyncio +from typing import TYPE_CHECKING +from unittest.mock import AsyncMock + +import pytest + +from noteflow.grpc.mixins._task_callbacks import create_job_done_callback + +if TYPE_CHECKING: + from collections.abc import Generator + +# Test constants +JOB_ID_1 = "job-001" +"""Test job identifier for primary job.""" + +JOB_ID_2 = "job-002" +"""Test job identifier for secondary job.""" + +TASK_ERROR_MESSAGE = "Processing failed unexpectedly" +"""Error message for simulated task failures.""" + + +@pytest.fixture +def tasks_dict() -> dict[str, asyncio.Task[None]]: + """Create empty tasks dictionary for tracking active tasks.""" + return {} + + +@pytest.fixture +def mock_mark_failed() -> AsyncMock: + """Create mock mark_failed coroutine function.""" + return AsyncMock() + + +@pytest.fixture +def event_loop_fixture() -> Generator[asyncio.AbstractEventLoop, None, None]: + """Create and manage event loop for synchronous callback tests.""" + loop = asyncio.new_event_loop() + asyncio.set_event_loop(loop) + yield loop + loop.close() + + +def _create_completed_task(loop: asyncio.AbstractEventLoop) -> asyncio.Task[None]: + """Create a successfully completed task.""" + async def successful() -> None: + pass + task = loop.create_task(successful()) + loop.run_until_complete(task) + return task + + +def _create_failed_task( + loop: asyncio.AbstractEventLoop, error_msg: str = TASK_ERROR_MESSAGE +) -> asyncio.Task[None]: + """Create a task that failed with ValueError.""" + async def failing() -> None: + raise ValueError(error_msg) + task = loop.create_task(failing()) + try: + loop.run_until_complete(task) + except ValueError: + pass + return task + + +def _create_cancelled_task(loop: asyncio.AbstractEventLoop) -> asyncio.Task[None]: + """Create a cancelled task without using sleep.""" + async def cancellable() -> None: + # Use an Event instead of sleep to make task cancellable + event = asyncio.Event() + await event.wait() + task = loop.create_task(cancellable()) + task.cancel() + try: + loop.run_until_complete(task) + except asyncio.CancelledError: + pass + return task + + +def _run_pending_callbacks(loop: asyncio.AbstractEventLoop) -> None: + """Run any pending callbacks on the loop.""" + loop.run_until_complete(asyncio.ensure_future(asyncio.sleep(0), loop=loop)) + + +class TestCallbackCreation: + """Tests for create_job_done_callback factory function.""" + + def test_returns_callable( + self, + tasks_dict: dict[str, asyncio.Task[None]], + mock_mark_failed: AsyncMock, + ) -> None: + """Verify factory returns a callable callback.""" + callback = create_job_done_callback(JOB_ID_1, tasks_dict, mock_mark_failed) + + assert callable(callback), "Factory should return a callable" + + def test_callback_accepts_task_argument( + self, + tasks_dict: dict[str, asyncio.Task[None]], + mock_mark_failed: AsyncMock, + event_loop_fixture: asyncio.AbstractEventLoop, + ) -> None: + """Verify callback signature accepts asyncio.Task.""" + task = _create_completed_task(event_loop_fixture) + callback = create_job_done_callback(JOB_ID_1, tasks_dict, mock_mark_failed) + + callback(task) # Should not raise + + +class TestTaskDictCleanupOnSuccess: + """Tests for tasks_dict cleanup on successful completion.""" + + def test_removes_job_on_success( + self, + tasks_dict: dict[str, asyncio.Task[None]], + mock_mark_failed: AsyncMock, + event_loop_fixture: asyncio.AbstractEventLoop, + ) -> None: + """Verify job is removed from tasks_dict on successful completion.""" + task = _create_completed_task(event_loop_fixture) + tasks_dict[JOB_ID_1] = task + + callback = create_job_done_callback(JOB_ID_1, tasks_dict, mock_mark_failed) + callback(task) + + assert JOB_ID_1 not in tasks_dict, ( + "Job should be removed from tasks_dict after completion" + ) + + +class TestTaskDictCleanupOnFailure: + """Tests for tasks_dict cleanup on task failure.""" + + def test_removes_job_on_failure( + self, + tasks_dict: dict[str, asyncio.Task[None]], + mock_mark_failed: AsyncMock, + event_loop_fixture: asyncio.AbstractEventLoop, + ) -> None: + """Verify job is removed from tasks_dict on task failure.""" + task = _create_failed_task(event_loop_fixture) + tasks_dict[JOB_ID_1] = task + + callback = create_job_done_callback(JOB_ID_1, tasks_dict, mock_mark_failed) + callback(task) + + assert JOB_ID_1 not in tasks_dict, ( + "Job should be removed from tasks_dict after failure" + ) + + +class TestTaskDictMissingJob: + """Tests for handling missing job in tasks_dict.""" + + def test_handles_missing_job_gracefully( + self, + tasks_dict: dict[str, asyncio.Task[None]], + mock_mark_failed: AsyncMock, + event_loop_fixture: asyncio.AbstractEventLoop, + ) -> None: + """Verify callback handles case where job is not in tasks_dict.""" + task = _create_completed_task(event_loop_fixture) + # Intentionally NOT adding to tasks_dict + + callback = create_job_done_callback(JOB_ID_1, tasks_dict, mock_mark_failed) + + callback(task) # Should not raise + + +class TestSuccessfulTaskNoMarkFailed: + """Tests verifying mark_failed is not called on success.""" + + def test_mark_failed_not_called_on_success( + self, + tasks_dict: dict[str, asyncio.Task[None]], + mock_mark_failed: AsyncMock, + event_loop_fixture: asyncio.AbstractEventLoop, + ) -> None: + """Verify mark_failed is not called when task succeeds.""" + task = _create_completed_task(event_loop_fixture) + + callback = create_job_done_callback(JOB_ID_1, tasks_dict, mock_mark_failed) + callback(task) + + mock_mark_failed.assert_not_called() + + +class TestCancelledTaskNoMarkFailed: + """Tests verifying mark_failed is not called on cancellation.""" + + def test_mark_failed_not_called_on_cancel( + self, + tasks_dict: dict[str, asyncio.Task[None]], + mock_mark_failed: AsyncMock, + event_loop_fixture: asyncio.AbstractEventLoop, + ) -> None: + """Verify mark_failed is not called when task is cancelled.""" + task = _create_cancelled_task(event_loop_fixture) + + callback = create_job_done_callback(JOB_ID_1, tasks_dict, mock_mark_failed) + callback(task) + + mock_mark_failed.assert_not_called() + + +class TestCancelledTaskCleanup: + """Tests for cancelled task cleanup.""" + + def test_removes_cancelled_job_from_tasks_dict( + self, + tasks_dict: dict[str, asyncio.Task[None]], + mock_mark_failed: AsyncMock, + event_loop_fixture: asyncio.AbstractEventLoop, + ) -> None: + """Verify cancelled job is removed from tasks_dict.""" + task = _create_cancelled_task(event_loop_fixture) + tasks_dict[JOB_ID_1] = task + + callback = create_job_done_callback(JOB_ID_1, tasks_dict, mock_mark_failed) + callback(task) + + assert JOB_ID_1 not in tasks_dict, ( + "Cancelled job should be removed from tasks_dict" + ) + + +class TestFailedTaskSchedulesMarkFailed: + """Tests for mark_failed scheduling on task failure.""" + + def test_schedules_mark_failed_on_exception( + self, + tasks_dict: dict[str, asyncio.Task[None]], + mock_mark_failed: AsyncMock, + event_loop_fixture: asyncio.AbstractEventLoop, + ) -> None: + """Verify mark_failed is scheduled when task raises exception.""" + task = _create_failed_task(event_loop_fixture) + + callback = create_job_done_callback(JOB_ID_1, tasks_dict, mock_mark_failed) + callback(task) + _run_pending_callbacks(event_loop_fixture) + + mock_mark_failed.assert_called_once() + + +class TestMarkFailedReceivesJobId: + """Tests for mark_failed receiving correct job_id.""" + + def test_receives_correct_job_id( + self, + tasks_dict: dict[str, asyncio.Task[None]], + mock_mark_failed: AsyncMock, + event_loop_fixture: asyncio.AbstractEventLoop, + ) -> None: + """Verify mark_failed is called with correct job_id.""" + task = _create_failed_task(event_loop_fixture) + + callback = create_job_done_callback(JOB_ID_1, tasks_dict, mock_mark_failed) + callback(task) + _run_pending_callbacks(event_loop_fixture) + + call_args = mock_mark_failed.call_args + assert call_args[0][0] == JOB_ID_1, ( + f"mark_failed should receive job_id '{JOB_ID_1}'" + ) + + +class TestMarkFailedReceivesErrorMessage: + """Tests for mark_failed receiving error message.""" + + def test_receives_error_message( + self, + tasks_dict: dict[str, asyncio.Task[None]], + mock_mark_failed: AsyncMock, + event_loop_fixture: asyncio.AbstractEventLoop, + ) -> None: + """Verify mark_failed is called with error message string.""" + task = _create_failed_task(event_loop_fixture) + + callback = create_job_done_callback(JOB_ID_1, tasks_dict, mock_mark_failed) + callback(task) + _run_pending_callbacks(event_loop_fixture) + + call_args = mock_mark_failed.call_args + assert TASK_ERROR_MESSAGE in call_args[0][1], ( + f"mark_failed should receive error message containing '{TASK_ERROR_MESSAGE}'" + ) + + +class TestVariousExceptionTypes: + """Tests for handling various exception types.""" + + @pytest.mark.parametrize( + ("exception_type", "error_prefix"), + [ + pytest.param(ValueError, "ValueError", id="value_error"), + pytest.param(RuntimeError, "RuntimeError", id="runtime_error"), + pytest.param(TypeError, "TypeError", id="type_error"), + ], + ) + def test_handles_exception_type( + self, + tasks_dict: dict[str, asyncio.Task[None]], + mock_mark_failed: AsyncMock, + event_loop_fixture: asyncio.AbstractEventLoop, + exception_type: type[Exception], + error_prefix: str, + ) -> None: + """Verify callback handles various exception types correctly.""" + error_msg = f"{error_prefix} test error" + + async def failing() -> None: + raise exception_type(error_msg) + + task = event_loop_fixture.create_task(failing()) + try: + event_loop_fixture.run_until_complete(task) + except exception_type: + pass + + callback = create_job_done_callback(JOB_ID_1, tasks_dict, mock_mark_failed) + callback(task) + _run_pending_callbacks(event_loop_fixture) + + assert mock_mark_failed.called, f"mark_failed should be called for {error_prefix}" + + +class TestClosedLoopHandling: + """Tests for handling RuntimeError when event loop is closed.""" + + def test_does_not_raise_on_closed_loop( + self, + tasks_dict: dict[str, asyncio.Task[None]], + mock_mark_failed: AsyncMock, + ) -> None: + """Verify callback does not raise when loop is closed. + + Note: mark_failed IS called to create the coroutine, but the + task scheduling fails. The callback gracefully handles this. + """ + loop = asyncio.new_event_loop() + asyncio.set_event_loop(loop) + task = _create_failed_task(loop) + loop.close() + + callback = create_job_done_callback(JOB_ID_1, tasks_dict, mock_mark_failed) + + # Key assertion: should not raise even with closed loop + callback(task) + + +class TestMultipleJobsIndependence: + """Tests for multiple job independence.""" + + def test_first_callback_removes_only_first_job( + self, + tasks_dict: dict[str, asyncio.Task[None]], + mock_mark_failed: AsyncMock, + event_loop_fixture: asyncio.AbstractEventLoop, + ) -> None: + """Verify first callback only removes first job.""" + task1 = _create_completed_task(event_loop_fixture) + task2 = _create_completed_task(event_loop_fixture) + tasks_dict[JOB_ID_1] = task1 + tasks_dict[JOB_ID_2] = task2 + + callback1 = create_job_done_callback(JOB_ID_1, tasks_dict, mock_mark_failed) + callback1(task1) + + assert JOB_ID_1 not in tasks_dict, "Job 1 should be removed" + assert JOB_ID_2 in tasks_dict, "Job 2 should still be present" + + def test_second_callback_removes_second_job( + self, + tasks_dict: dict[str, asyncio.Task[None]], + mock_mark_failed: AsyncMock, + event_loop_fixture: asyncio.AbstractEventLoop, + ) -> None: + """Verify second callback removes second job.""" + task1 = _create_completed_task(event_loop_fixture) + task2 = _create_completed_task(event_loop_fixture) + tasks_dict[JOB_ID_1] = task1 + tasks_dict[JOB_ID_2] = task2 + + callback1 = create_job_done_callback(JOB_ID_1, tasks_dict, mock_mark_failed) + callback2 = create_job_done_callback(JOB_ID_2, tasks_dict, mock_mark_failed) + callback1(task1) + callback2(task2) + + assert JOB_ID_2 not in tasks_dict, "Job 2 should be removed" diff --git a/tests/grpc/test_timestamp_converters.py b/tests/grpc/test_timestamp_converters.py index 1a8d66b..fd828a6 100644 --- a/tests/grpc/test_timestamp_converters.py +++ b/tests/grpc/test_timestamp_converters.py @@ -7,7 +7,7 @@ from datetime import UTC, datetime, timedelta, timezone import pytest from google.protobuf.timestamp_pb2 import Timestamp -from noteflow.grpc._mixins.converters import ( +from noteflow.grpc.mixins.converters import ( datetime_to_epoch_seconds, datetime_to_iso_string, datetime_to_proto_timestamp, diff --git a/tests/grpc/test_webhooks_mixin.py b/tests/grpc/test_webhooks_mixin.py index 6a2efdd..bcfb380 100644 --- a/tests/grpc/test_webhooks_mixin.py +++ b/tests/grpc/test_webhooks_mixin.py @@ -23,8 +23,8 @@ from noteflow.domain.webhooks import ( WebhookDelivery, WebhookEventType, ) -from noteflow.grpc._mixins._types import GrpcContext -from noteflow.grpc._mixins.webhooks import WebhooksMixin +from noteflow.grpc.mixins._types import GrpcContext +from noteflow.grpc.mixins.webhooks import WebhooksMixin from noteflow.grpc.proto import noteflow_pb2 # ============================================================================ @@ -525,20 +525,22 @@ class TestDeleteWebhook: assert response.success is True, "Should return success=true" mock_webhook_repo.delete.assert_called_once_with(webhook_id) - async def test_delete_returns_false_for_nonexistent( + async def test_delete_aborts_for_nonexistent( self, webhooks_servicer: MockServicerHost, mock_webhook_repo: AsyncMock, mock_grpc_context: MagicMock, ) -> None: - """DeleteWebhook returns success=false for nonexistent webhook.""" + """DeleteWebhook aborts with NOT_FOUND for nonexistent webhook.""" mock_webhook_repo.delete.return_value = False webhook_id = uuid4() request = noteflow_pb2.DeleteWebhookRequest(webhook_id=str(webhook_id)) - response = await webhooks_servicer.DeleteWebhook(request, mock_grpc_context) - assert response.success is False, "Should return success=false" + with pytest.raises(AssertionError, match="Unreachable"): + await webhooks_servicer.DeleteWebhook(request, mock_grpc_context) + + mock_grpc_context.abort.assert_called_once() async def test_delete_rejects_invalid_id( self, diff --git a/tests/infrastructure/auth/test_oidc_registry.py b/tests/infrastructure/auth/test_oidc_registry.py index 7fa7e53..118c9ea 100644 --- a/tests/infrastructure/auth/test_oidc_registry.py +++ b/tests/infrastructure/auth/test_oidc_registry.py @@ -360,15 +360,17 @@ class TestOidcAuthService: assert isinstance(warnings, list), "warnings should be a list" def test_get_preset_options(self, auth_service: OidcAuthService) -> None: - """Verify preset options retrieval.""" + """Verify preset options retrieval filters out custom preset.""" options = auth_service.get_preset_options() - assert len(options) == len(OidcProviderPreset), "should have all preset options" + # get_preset_options filters out 'custom' preset template + expected_count = len(OidcProviderPreset) - 1 + assert len(options) == expected_count, "should have all presets except custom" preset_values = {opt["preset"] for opt in options} assert "authentik" in preset_values, "should include authentik" assert "authelia" in preset_values, "should include authelia" assert "keycloak" in preset_values, "should include keycloak" - assert "custom" in preset_values, "should include custom" + assert "custom" not in preset_values, "should NOT include custom (filtered out)" @pytest.mark.asyncio async def test_refresh_all_discovery_returns_both( diff --git a/tests/infrastructure/metrics/test_infrastructure_metrics.py b/tests/infrastructure/metrics/test_infrastructure_metrics.py new file mode 100644 index 0000000..1851bb5 --- /dev/null +++ b/tests/infrastructure/metrics/test_infrastructure_metrics.py @@ -0,0 +1,232 @@ +"""Tests for infrastructure metrics tracking.""" + +from __future__ import annotations + +import time + +import pytest + +from noteflow.infrastructure.metrics import ( + InfrastructureMetrics, + InfrastructureStats, + get_infrastructure_metrics, + reset_infrastructure_metrics, +) + + +@pytest.fixture(autouse=True) +def reset_metrics_fixture() -> None: + """Reset metrics singleton before each test.""" + reset_infrastructure_metrics() + + +def test_record_fallback_increments_count() -> None: + """Test that recording a fallback increments the count.""" + metrics = InfrastructureMetrics() + + metrics.record_fallback("ollama_settings", "ImportError") + + stats = metrics.get_infrastructure_stats() + assert stats.total_fallbacks == 1, "Should record one fallback" + assert stats.fallbacks_by_component == { + "ollama_settings": 1 + }, "Should track fallback by component" + + +def test_record_fallback_multiple_components() -> None: + """Test that fallbacks are tracked by component.""" + metrics = InfrastructureMetrics() + + metrics.record_fallback("ollama_settings", "ImportError") + metrics.record_fallback("calendar_adapter", "ValueError") + metrics.record_fallback("ollama_settings", "KeyError") + + stats = metrics.get_infrastructure_stats() + assert stats.total_fallbacks == 3, "Should count all fallbacks" + assert stats.fallbacks_by_component == { + "ollama_settings": 2, + "calendar_adapter": 1, + }, "Should aggregate by component" + + +def test_record_buffer_overflow_tracks_samples() -> None: + """Test that buffer overflows track dropped samples.""" + metrics = InfrastructureMetrics() + + first_drop = 1024 + second_drop = 512 + expected_total = first_drop + second_drop + + metrics.record_buffer_overflow("partial_audio", first_drop) + metrics.record_buffer_overflow("partial_audio", second_drop) + + stats = metrics.get_infrastructure_stats() + assert stats.total_buffer_overflows == 2, "Should count both overflows" + assert stats.samples_dropped == expected_total, "Should sum dropped samples" + + +def test_record_provider_unavailable_tracks_providers() -> None: + """Test that provider unavailability is tracked.""" + metrics = InfrastructureMetrics() + + metrics.record_provider_unavailable("sounddevice", "no_loopback_device") + metrics.record_provider_unavailable("pywinctl", "import_error") + + stats = metrics.get_infrastructure_stats() + assert stats.unavailable_providers == {"sounddevice", "pywinctl"} + + +def test_record_provider_unavailable_deduplicates() -> None: + """Test that provider unavailability deduplicates providers.""" + metrics = InfrastructureMetrics() + + metrics.record_provider_unavailable("sounddevice", "no_loopback_device") + metrics.record_provider_unavailable("sounddevice", "stream_failed") + + stats = metrics.get_infrastructure_stats() + assert stats.unavailable_providers == {"sounddevice"} + + +def test_get_stats_returns_empty_when_no_events() -> None: + """Test that empty stats are returned when no events recorded.""" + metrics = InfrastructureMetrics() + + stats = metrics.get_infrastructure_stats() + expected_empty = InfrastructureStats.empty() + + # Verify all fields match empty stats + assert ( + stats.total_fallbacks == expected_empty.total_fallbacks + and stats.fallbacks_by_component == expected_empty.fallbacks_by_component + and stats.total_buffer_overflows == expected_empty.total_buffer_overflows + and stats.samples_dropped == expected_empty.samples_dropped + and stats.unavailable_providers == expected_empty.unavailable_providers + ), f"Empty stats mismatch: {stats} != {expected_empty}" + + +def test_singleton_returns_same_instance() -> None: + """Test that get_infrastructure_metrics returns singleton.""" + metrics1 = get_infrastructure_metrics() + metrics2 = get_infrastructure_metrics() + + assert metrics1 is metrics2 + + +def test_reset_clears_singleton() -> None: + """Test that reset_infrastructure_metrics clears state.""" + metrics1 = get_infrastructure_metrics() + metrics1.record_fallback("test", "Error") + + reset_infrastructure_metrics() + metrics2 = get_infrastructure_metrics() + + assert metrics1 is not metrics2, "Reset should create new instance" + assert metrics2.get_infrastructure_stats().total_fallbacks == 0, "New instance should be empty" + + +def test_rolling_window_pruning() -> None: + """Test that old metrics are pruned from rolling window.""" + from noteflow.infrastructure.metrics.infrastructure_metrics import ( + METRIC_WINDOW_SECONDS, + ) + + metrics = InfrastructureMetrics() + + # Record metric + metrics.record_fallback("test", "Error") + assert metrics.get_infrastructure_stats().total_fallbacks == 1, "Should record initial metric" + + # Advance time beyond window (monkey-patch time.time) + original_time = time.time + try: + time.time = lambda: original_time() + METRIC_WINDOW_SECONDS + 1 # type: ignore[method-assign] + + # Metric should be pruned + assert metrics.get_infrastructure_stats().total_fallbacks == 0, "Old metrics should be pruned" + finally: + time.time = original_time # type: ignore[method-assign] + + +def test_infrastructure_stats_empty_factory() -> None: + """Test InfrastructureStats.empty() factory.""" + stats = InfrastructureStats.empty() + + expected_zero = 0 + expected_empty_dict: dict[str, int] = {} + expected_empty_set: set[str] = set() + + assert stats.total_fallbacks == expected_zero, "Fallbacks should be zero" + assert stats.fallbacks_by_component == expected_empty_dict, "Component dict should be empty" + assert stats.total_buffer_overflows == expected_zero, "Overflows should be zero" + assert stats.samples_dropped == expected_zero, "Dropped samples should be zero" + assert stats.unavailable_providers == expected_empty_set, "Providers set should be empty" + + +def test_thread_safety_fallbacks(monkeypatch: pytest.MonkeyPatch) -> None: + """Test that fallback recording is thread-safe.""" + import threading + + metrics = InfrastructureMetrics() + iterations = 100 + + def record_fallbacks() -> None: + for _ in range(iterations): + metrics.record_fallback("component", "Error") + + threads = [threading.Thread(target=record_fallbacks) for _ in range(10)] + for thread in threads: + thread.start() + for thread in threads: + thread.join() + + stats = metrics.get_infrastructure_stats() + assert stats.total_fallbacks == 10 * iterations + + +def test_thread_safety_overflows(monkeypatch: pytest.MonkeyPatch) -> None: + """Test that overflow recording is thread-safe.""" + import threading + + metrics = InfrastructureMetrics() + iterations = 100 + num_threads = 10 + samples_per_overflow = 10 + + def record_overflows() -> None: + for _ in range(iterations): + metrics.record_buffer_overflow("buffer", samples_per_overflow) + + threads = [threading.Thread(target=record_overflows) for _ in range(num_threads)] + for thread in threads: + thread.start() + for thread in threads: + thread.join() + + expected_overflows = num_threads * iterations + expected_dropped = expected_overflows * samples_per_overflow + + stats = metrics.get_infrastructure_stats() + assert stats.total_buffer_overflows == expected_overflows, "Should count all overflows" + assert stats.samples_dropped == expected_dropped, "Should sum all dropped samples" + + +def test_thread_safety_providers(monkeypatch: pytest.MonkeyPatch) -> None: + """Test that provider recording is thread-safe.""" + import threading + + metrics = InfrastructureMetrics() + iterations = 100 + + def record_providers() -> None: + for i in range(iterations): + metrics.record_provider_unavailable(f"provider_{i % 5}", "reason") + + threads = [threading.Thread(target=record_providers) for _ in range(10)] + for thread in threads: + thread.start() + for thread in threads: + thread.join() + + stats = metrics.get_infrastructure_stats() + # Should have 5 unique providers (provider_0 through provider_4) + assert len(stats.unavailable_providers) == 5 diff --git a/tests/infrastructure/observability/test_log_buffer.py b/tests/infrastructure/observability/test_log_buffer.py index fab06a8..77928e5 100644 --- a/tests/infrastructure/observability/test_log_buffer.py +++ b/tests/infrastructure/observability/test_log_buffer.py @@ -142,31 +142,38 @@ class TestLogBuffer: entries[2].message == "Message 2" ), "oldest entry should be Message 2 (Message 1 evicted)" - def test_buffer_filters_by_level(self) -> None: - """LogBuffer filters entries by level.""" + @pytest.mark.parametrize( + ("log_entries", "filter_kwargs", "expected_message"), + [ + pytest.param( + [("info", "app", "Info message"), ("error", "app", "Error message")], + {"level": "error"}, + "Error message", + id="filter-by-level", + ), + pytest.param( + [("info", "app", "App message"), ("info", "api", "API message")], + {"source": "api"}, + "API message", + id="filter-by-source", + ), + ], + ) + def test_buffer_filters( + self, + log_entries: list[tuple[str, str, str]], + filter_kwargs: dict[str, str], + expected_message: str, + ) -> None: + """LogBuffer filters entries by specified criteria.""" buffer = LogBuffer() - buffer.log("info", "app", "Info message") - buffer.log("error", "app", "Error message") - buffer.log("info", "app", "Another info") + for level, source, message in log_entries: + buffer.log(level, source, message) - entries = buffer.get_recent(limit=DEFAULT_LIMIT, level="error") - assert len(entries) == 1, "filter should return only error entries" - assert ( - entries[0].message == "Error message" - ), "filtered entry should be error message" + entries = buffer.get_recent(limit=DEFAULT_LIMIT, **filter_kwargs) - def test_buffer_filters_by_source(self) -> None: - """LogBuffer filters entries by source.""" - buffer = LogBuffer() - buffer.log("info", "app", "App message") - buffer.log("info", "api", "API message") - buffer.log("info", "app", "Another app") - - entries = buffer.get_recent(limit=DEFAULT_LIMIT, source="api") - assert len(entries) == 1, "filter should return only api entries" - assert ( - entries[0].message == "API message" - ), "filtered entry should be API message" + assert len(entries) == 1, "filter should return exactly one entry" + assert entries[0].message == expected_message, "filtered entry should match expected" class TestLogBufferHandler: diff --git a/tests/infrastructure/observability/test_usage.py b/tests/infrastructure/observability/test_usage.py index 5a56d2f..1391afe 100644 --- a/tests/infrastructure/observability/test_usage.py +++ b/tests/infrastructure/observability/test_usage.py @@ -168,7 +168,7 @@ class TestCreateUsageEventSink: ) -> None: """Factory returns LoggingUsageEventSink when OTel unavailable.""" monkeypatch.setattr( - "noteflow.infrastructure.observability.usage.check_otel_available", + "noteflow.infrastructure.observability.usage.usage.check_otel_available", lambda: False, ) sink = create_usage_event_sink(use_otel=True, fallback_to_logging=True) diff --git a/tests/infrastructure/persistence/test_logging_persistence.py b/tests/infrastructure/persistence/test_logging_persistence.py index 09d7ca1..f2a4d35 100644 --- a/tests/infrastructure/persistence/test_logging_persistence.py +++ b/tests/infrastructure/persistence/test_logging_persistence.py @@ -28,22 +28,22 @@ class TestUnitOfWorkLogging: """UoW logs session start event name on __aenter__.""" # The logger.debug call with uow_session_started should be present # Verify the logger module has the expected log call pattern - from noteflow.infrastructure.persistence import unit_of_work + from noteflow.infrastructure.persistence.unit_of_work import _context_mixin # Check the logger is imported correctly - assert hasattr(unit_of_work, "logger") + assert hasattr(_context_mixin, "logger") def test_uow_commit_logged(self) -> None: """UoW logs commit event name with duration.""" - from noteflow.infrastructure.persistence import unit_of_work + from noteflow.infrastructure.persistence.unit_of_work import _context_mixin - assert hasattr(unit_of_work, "logger") + assert hasattr(_context_mixin, "logger") def test_uow_session_closed_logged(self) -> None: """UoW logs session close event name on __aexit__.""" - from noteflow.infrastructure.persistence import unit_of_work + from noteflow.infrastructure.persistence.unit_of_work import _context_mixin - assert hasattr(unit_of_work, "logger") + assert hasattr(_context_mixin, "logger") class TestRepositoryLogging: diff --git a/tests/infrastructure/persistence/test_migrations.py b/tests/infrastructure/persistence/test_migrations.py index b590c4e..130d05d 100644 --- a/tests/infrastructure/persistence/test_migrations.py +++ b/tests/infrastructure/persistence/test_migrations.py @@ -53,10 +53,6 @@ def _find_all_migration_files() -> list[Path]: class TestMigrationStructure: """Verify migration file structure.""" - def test_migrations_directory_exists(self) -> None: - """Migrations directory should exist.""" - assert MIGRATIONS_DIR.exists(), f"Missing migrations directory: {MIGRATIONS_DIR}" - def test_all_migrations_have_revision(self) -> None: """Each migration should have a revision identifier.""" missing_revision = [ diff --git a/tests/infrastructure/triggers/test_calendar.py b/tests/infrastructure/triggers/test_calendar.py index 3967953..1d336a5 100644 --- a/tests/infrastructure/triggers/test_calendar.py +++ b/tests/infrastructure/triggers/test_calendar.py @@ -91,28 +91,42 @@ class TestCalendarProviderGetSignal: provider = CalendarProvider(_settings(events=[])) assert provider.get_signal() is None, "no events should return None" - def test_event_in_lookahead_window(self) -> None: - """Event starting within lookahead window should trigger signal.""" - event = _event(minutes_from_now=3) - provider = CalendarProvider(_settings(lookahead_minutes=5, events=[event])) + @pytest.mark.parametrize( + ("minutes_offset", "duration", "settings_kwargs", "window_type"), + [ + pytest.param( + 3, + 30, + {"lookahead_minutes": 5}, + "lookahead", + id="lookahead-window", + ), + pytest.param( + -3, + 60, + {"lookbehind_minutes": 5}, + "lookbehind", + id="lookbehind-window", + ), + ], + ) + def test_event_in_window_triggers_signal( + self, + minutes_offset: int, + duration: int, + settings_kwargs: dict[str, int], + window_type: str, + ) -> None: + """Events within configured time windows trigger signals.""" + event = _event(minutes_from_now=minutes_offset, duration_minutes=duration) + provider = CalendarProvider(_settings(**settings_kwargs, events=[event])) signal = provider.get_signal() - assert signal is not None, "Expected signal for event in lookahead window" + assert signal is not None, f"Expected signal for event in {window_type} window" assert signal.source == TriggerSource.CALENDAR, "Signal source should be CALENDAR" - assert signal.weight == approx_float(0.3), "Signal weight should match configured weight" assert signal.app_name == "Test Meeting", "Signal app_name should be event title" - def test_event_in_lookbehind_window(self) -> None: - """Event that started recently should trigger signal.""" - event = _event(minutes_from_now=-3, duration_minutes=60) - provider = CalendarProvider(_settings(lookbehind_minutes=5, events=[event])) - - signal = provider.get_signal() - - assert signal is not None, "recently started event should trigger" - assert signal.app_name == "Test Meeting", "signal should include event title" - def test_event_outside_window_returns_none(self) -> None: """Event outside lookahead/lookbehind window should not trigger.""" event = _event(minutes_from_now=20) diff --git a/tests/infrastructure/triggers/test_foreground_app.py b/tests/infrastructure/triggers/test_foreground_app.py index b7b46f3..4884eb2 100644 --- a/tests/infrastructure/triggers/test_foreground_app.py +++ b/tests/infrastructure/triggers/test_foreground_app.py @@ -78,7 +78,7 @@ def test_foreground_app_provider_suppressed(monkeypatch: pytest.MonkeyPatch) -> def test_foreground_app_provider_unavailable(monkeypatch: pytest.MonkeyPatch) -> None: """Unavailable provider should report disabled.""" provider = ForegroundAppProvider(_settings()) - monkeypatch.setattr(provider, "_is_available", lambda: False) + monkeypatch.setattr(provider, "_is_provider_available", lambda: False) assert provider.is_enabled() is False, "Unavailable provider should report disabled" @@ -139,7 +139,7 @@ def test_foreground_app_provider_no_window_returns_none( _install_pywinctl(monkeypatch, None) provider = ForegroundAppProvider(_settings()) # Force availability check to succeed via method patch - monkeypatch.setattr(provider, "_is_available", _make_is_available_true) + monkeypatch.setattr(provider, "_is_provider_available", _make_is_available_true) assert provider.get_signal() is None, "No active window should return None" @@ -150,7 +150,7 @@ def test_foreground_app_provider_empty_title_returns_none( """Provider should return None when window title is empty string.""" _install_pywinctl(monkeypatch, "") provider = ForegroundAppProvider(_settings()) - monkeypatch.setattr(provider, "_is_available", _make_is_available_true) + monkeypatch.setattr(provider, "_is_provider_available", _make_is_available_true) assert provider.get_signal() is None, "Empty window title should return None" @@ -161,7 +161,7 @@ def test_foreground_app_provider_non_meeting_app_returns_none( """Provider should return None when foreground app is not a meeting app.""" _install_pywinctl(monkeypatch, "Firefox Browser") provider = ForegroundAppProvider(_settings(meeting_apps={"zoom", "teams"})) - monkeypatch.setattr(provider, "_is_available", _make_is_available_true) + monkeypatch.setattr(provider, "_is_provider_available", _make_is_available_true) assert provider.get_signal() is None, "Non-meeting app should return None" @@ -215,7 +215,7 @@ def test_foreground_app_provider_case_insensitive_matching( """Provider should match meeting apps case-insensitively.""" _install_pywinctl(monkeypatch, "ZOOM MEETING - Conference Room") provider = ForegroundAppProvider(_settings(meeting_apps={"zoom"})) - monkeypatch.setattr(provider, "_is_available", _make_is_available_true) + monkeypatch.setattr(provider, "_is_provider_available", _make_is_available_true) signal = provider.get_signal() diff --git a/tests/integration/test_e2e_export.py b/tests/integration/test_e2e_export.py index 8de1c1b..0f714f4 100644 --- a/tests/integration/test_e2e_export.py +++ b/tests/integration/test_e2e_export.py @@ -20,7 +20,7 @@ from uuid import uuid4 import grpc import pytest -from noteflow.application.services.export_service import ExportFormat, ExportService +from noteflow.application.services.export import ExportFormat, ExportService from noteflow.domain.entities import Meeting, Segment from noteflow.domain.value_objects import MeetingId from noteflow.grpc.proto import noteflow_pb2 diff --git a/tests/integration/test_e2e_ner.py b/tests/integration/test_e2e_ner.py index e950743..04fe6dd 100644 --- a/tests/integration/test_e2e_ner.py +++ b/tests/integration/test_e2e_ner.py @@ -19,7 +19,7 @@ from uuid import UUID, uuid4 import grpc import pytest -from noteflow.application.services.ner_service import NerService +from noteflow.application.services.ner import NerService from noteflow.domain.entities import Meeting, Segment from noteflow.domain.entities.named_entity import EntityCategory, NamedEntity from noteflow.domain.value_objects import MeetingId @@ -38,7 +38,7 @@ def mock_feature_flags() -> Generator[MagicMock, None, None]: """Mock feature flags to enable NER for all tests.""" mock_flags = MagicMock(ner_enabled=True) with patch( - "noteflow.application.services.ner_service.get_feature_flags", + "noteflow.application.services.ner.service.get_feature_flags", return_value=mock_flags, ): yield mock_flags diff --git a/tests/integration/test_e2e_streaming.py b/tests/integration/test_e2e_streaming.py index 394e93b..e9bf58c 100644 --- a/tests/integration/test_e2e_streaming.py +++ b/tests/integration/test_e2e_streaming.py @@ -26,7 +26,7 @@ from numpy.typing import NDArray from noteflow.config.constants import DEFAULT_SAMPLE_RATE from noteflow.domain.entities import Meeting, Segment from noteflow.domain.value_objects import MeetingId, MeetingState -from noteflow.grpc._mixins.streaming import StreamingMixin +from noteflow.grpc.mixins.streaming import StreamingMixin from noteflow.grpc.proto import noteflow_pb2 from noteflow.grpc.service import NoteFlowServicer from noteflow.grpc.stream_state import MeetingStreamState @@ -37,7 +37,7 @@ from noteflow.infrastructure.persistence.unit_of_work import SqlAlchemyUnitOfWor if TYPE_CHECKING: from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker -from noteflow.grpc._mixins._types import GrpcContext +from noteflow.grpc.mixins._types import GrpcContext from support.async_helpers import drain_async_gen, yield_control # Type alias for StreamTranscription method signature diff --git a/tests/integration/test_e2e_summarization.py b/tests/integration/test_e2e_summarization.py index dc91548..d573baa 100644 --- a/tests/integration/test_e2e_summarization.py +++ b/tests/integration/test_e2e_summarization.py @@ -20,7 +20,7 @@ import pytest from noteflow.domain.entities import ActionItem, KeyPoint, Meeting, Segment, Summary from noteflow.domain.summarization import SummarizationResult from noteflow.domain.value_objects import MeetingId -from noteflow.grpc._config import ServicesConfig +from noteflow.grpc.config.config import ServicesConfig from noteflow.grpc.proto import noteflow_pb2 from noteflow.grpc.service import NoteFlowServicer from noteflow.infrastructure.persistence.unit_of_work import SqlAlchemyUnitOfWork diff --git a/tests/integration/test_grpc_servicer_database.py b/tests/integration/test_grpc_servicer_database.py index 2ff8cb5..1dce8fe 100644 --- a/tests/integration/test_grpc_servicer_database.py +++ b/tests/integration/test_grpc_servicer_database.py @@ -25,7 +25,7 @@ import pytest from noteflow.domain.entities import Meeting, Segment from noteflow.domain.entities.named_entity import EntityCategory, NamedEntity from noteflow.domain.value_objects import MeetingId, MeetingState -from noteflow.grpc._config import ServicesConfig +from noteflow.grpc.config.config import ServicesConfig from noteflow.grpc.proto import noteflow_pb2 from noteflow.grpc.service import NoteFlowServicer from noteflow.infrastructure.persistence.repositories import DiarizationJob diff --git a/tests/integration/test_hf_token_grpc.py b/tests/integration/test_hf_token_grpc.py index e3fe49c..369e57a 100644 --- a/tests/integration/test_hf_token_grpc.py +++ b/tests/integration/test_hf_token_grpc.py @@ -8,7 +8,7 @@ from unittest.mock import MagicMock, patch import pytest -from noteflow.application.services.hf_token_service import HfTokenService, HfValidationResult +from noteflow.application.services.huggingface import HfTokenService, HfValidationResult from noteflow.grpc.proto import noteflow_pb2 from noteflow.grpc.service import NoteFlowServicer from noteflow.infrastructure.security.crypto import AesGcmCryptoBox diff --git a/tests/integration/test_recovery_service.py b/tests/integration/test_recovery_service.py index a0a7985..cceb5f9 100644 --- a/tests/integration/test_recovery_service.py +++ b/tests/integration/test_recovery_service.py @@ -260,9 +260,9 @@ class TestRecoveryServiceDiarizationJobRecovery: await uow.commit() recovery_service = RecoveryService(SqlAlchemyUnitOfWork(session_factory, meetings_dir)) - failed_count = await recovery_service.recover_crashed_diarization_jobs() + result = await recovery_service.recover_crashed_diarization_jobs() - assert failed_count == 1, "should fail exactly one queued job" + assert result.jobs_recovered == 1, "should fail exactly one queued job" async with SqlAlchemyUnitOfWork(session_factory, meetings_dir) as uow: retrieved = await uow.diarization_jobs.get(job.job_id) @@ -287,9 +287,9 @@ class TestRecoveryServiceDiarizationJobRecovery: await uow.commit() recovery_service = RecoveryService(SqlAlchemyUnitOfWork(session_factory, meetings_dir)) - failed_count = await recovery_service.recover_crashed_diarization_jobs() + result = await recovery_service.recover_crashed_diarization_jobs() - assert failed_count == 1, "should fail exactly one running diarization job" + assert result.jobs_recovered == 1, "should fail exactly one running diarization job" async def test_ignores_completeddiarization_jobs( self, session_factory: async_sessionmaker[AsyncSession], meetings_dir: Path @@ -309,9 +309,9 @@ class TestRecoveryServiceDiarizationJobRecovery: await uow.commit() recovery_service = RecoveryService(SqlAlchemyUnitOfWork(session_factory, meetings_dir)) - failed_count = await recovery_service.recover_crashed_diarization_jobs() + result = await recovery_service.recover_crashed_diarization_jobs() - assert failed_count == 0, "should not fail any completed diarization jobs" + assert result.jobs_recovered == 0, "should not fail any completed diarization jobs" async with SqlAlchemyUnitOfWork(session_factory, meetings_dir) as uow: retrieved = await uow.diarization_jobs.get(job.job_id) @@ -336,9 +336,9 @@ class TestRecoveryServiceDiarizationJobRecovery: await uow.commit() recovery_service = RecoveryService(SqlAlchemyUnitOfWork(session_factory, meetings_dir)) - failed_count = await recovery_service.recover_crashed_diarization_jobs() + result = await recovery_service.recover_crashed_diarization_jobs() - assert failed_count == 0, "should not fail any already-failed diarization jobs" + assert result.jobs_recovered == 0, "should not fail any already-failed diarization jobs" async with SqlAlchemyUnitOfWork(session_factory, meetings_dir) as uow: retrieved = await uow.diarization_jobs.get(job.job_id) diff --git a/tests/integration/test_webhook_integration.py b/tests/integration/test_webhook_integration.py index 5e0b99e..621bf7e 100644 --- a/tests/integration/test_webhook_integration.py +++ b/tests/integration/test_webhook_integration.py @@ -13,7 +13,7 @@ from uuid import uuid4 import grpc import pytest -from noteflow.application.services.webhook_service import WebhookService +from noteflow.application.services.webhooks import WebhookService from noteflow.domain.entities import Meeting, Segment from noteflow.domain.webhooks import ( DeliveryResult, @@ -21,7 +21,7 @@ from noteflow.domain.webhooks import ( WebhookDelivery, WebhookEventType, ) -from noteflow.grpc._config import ServicesConfig +from noteflow.grpc.config.config import ServicesConfig from noteflow.grpc.proto import noteflow_pb2 from noteflow.grpc.service import NoteFlowServicer from noteflow.infrastructure.persistence.unit_of_work import SqlAlchemyUnitOfWork diff --git a/tests/quality/_detectors/__init__.py b/tests/quality/_detectors/__init__.py new file mode 100644 index 0000000..e3d33e4 --- /dev/null +++ b/tests/quality/_detectors/__init__.py @@ -0,0 +1,86 @@ +"""Shared quality rule detectors. + +These collectors power both baseline generation and quality tests. +""" + +from __future__ import annotations + +from tests.quality._detectors.code_smells import ( + collect_deep_nesting, + collect_feature_envy, + collect_god_classes, + collect_high_complexity, + collect_long_methods, + collect_long_parameter_lists, + collect_module_size_soft, +) +from tests.quality._detectors.stale_code import ( + collect_deprecated_patterns, + collect_orphaned_imports, + collect_stale_todos, +) +from tests.quality._detectors.test_smells import ( + collect_assertion_roulette, + collect_conditional_test_logic, + collect_duplicate_test_names, + collect_eager_tests, + collect_exception_handling, + collect_fixture_missing_type, + collect_fixture_scope_too_narrow, + collect_long_tests, + collect_magic_number_tests, + collect_raises_without_match, + collect_redundant_prints, + collect_sensitive_equality, + collect_sleepy_tests, + collect_unknown_tests, + collect_unused_fixtures, + get_fixture_scope, + get_fixtures, + get_module_level_fixtures, + get_test_methods, +) +from tests.quality._detectors.wrappers import ( + collect_alias_imports, + collect_passthrough_classes, + collect_redundant_type_aliases, + collect_thin_wrappers, + is_thin_wrapper, +) + +__all__ = [ + "collect_assertion_roulette", + "collect_conditional_test_logic", + "collect_duplicate_test_names", + "collect_eager_tests", + "collect_exception_handling", + "collect_fixture_missing_type", + "collect_fixture_scope_too_narrow", + "collect_long_tests", + "collect_magic_number_tests", + "collect_raises_without_match", + "collect_redundant_prints", + "collect_sensitive_equality", + "collect_sleepy_tests", + "collect_unknown_tests", + "collect_unused_fixtures", + "collect_deep_nesting", + "collect_feature_envy", + "collect_god_classes", + "collect_high_complexity", + "collect_long_methods", + "collect_long_parameter_lists", + "collect_module_size_soft", + "collect_deprecated_patterns", + "collect_orphaned_imports", + "collect_stale_todos", + "collect_alias_imports", + "collect_passthrough_classes", + "collect_redundant_type_aliases", + "collect_thin_wrappers", + "get_fixture_scope", + "get_fixtures", + "get_module_level_fixtures", + "get_test_methods", + "is_thin_wrapper", +] diff --git a/tests/quality/_detectors/code_smells.py b/tests/quality/_detectors/code_smells.py new file mode 100644 index 0000000..143cd0b --- /dev/null +++ b/tests/quality/_detectors/code_smells.py @@ -0,0 +1,369 @@ +"""Shared detectors for code smell quality rules.""" + +from __future__ import annotations + +import ast +from pathlib import Path + +from tests.quality._baseline import Violation +from tests.quality._helpers import ( + find_source_files, + parse_file_safe, + read_file_safe, + relative_path, +) + + +def _parse_tree(py_file: Path, parse_errors: list[str] | None) -> ast.AST | None: + tree, error = parse_file_safe(py_file) + if error is not None: + if parse_errors is not None: + parse_errors.append(error) + return None + return tree + + +def count_branches(node: ast.AST) -> int: + """Count decision points (branches) in an AST node.""" + count = 0 + for child in ast.walk(node): + if isinstance( + child, (ast.If, ast.While, ast.For, ast.AsyncFor, ast.And, ast.Or) + ): + count += 1 + elif isinstance(child, ast.comprehension): + count += 1 + count += len(child.ifs) + elif isinstance(child, ast.ExceptHandler): + count += 1 + return count + + +def count_nesting_depth(node: ast.AST, current_depth: int = 0) -> int: + """Calculate maximum nesting depth.""" + max_depth = current_depth + + nesting_nodes = ( + ast.If, + ast.While, + ast.For, + ast.AsyncFor, + ast.With, + ast.AsyncWith, + ast.Try, + ast.FunctionDef, + ast.AsyncFunctionDef, + ) + + for child in ast.iter_child_nodes(node): + if isinstance(child, nesting_nodes): + child_depth = count_nesting_depth(child, current_depth + 1) + else: + child_depth = count_nesting_depth(child, current_depth) + max_depth = max(max_depth, child_depth) + + return max_depth + + +def count_function_lines(node: ast.FunctionDef | ast.AsyncFunctionDef) -> int: + """Count lines in a function body.""" + return 0 if node.end_lineno is None else node.end_lineno - node.lineno + 1 + + +def collect_high_complexity( + *, + parse_errors: list[str] | None = None, + max_complexity: int = 12, +) -> list[Violation]: + """Collect high cyclomatic complexity violations.""" + violations: list[Violation] = [] + + for py_file in find_source_files(include_migrations=False): + tree = _parse_tree(py_file, parse_errors) + if tree is None: + continue + + rel_path = relative_path(py_file) + + for node in ast.walk(tree): + if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)): + complexity = count_branches(node) + 1 + if complexity > max_complexity: + violations.append( + Violation( + rule="high_complexity", + relative_path=rel_path, + identifier=node.name, + detail=f"complexity={complexity}", + ) + ) + + return violations + + +def collect_long_parameter_lists( + *, + parse_errors: list[str] | None = None, + max_params: int = 4, +) -> list[Violation]: + """Collect long parameter list violations.""" + violations: list[Violation] = [] + + for py_file in find_source_files(include_migrations=False): + tree = _parse_tree(py_file, parse_errors) + if tree is None: + continue + + rel_path = relative_path(py_file) + + for node in ast.walk(tree): + if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)): + args = node.args + total_params = ( + len(args.args) + len(args.posonlyargs) + len(args.kwonlyargs) + ) + if "self" in [a.arg for a in args.args]: + total_params -= 1 + if "cls" in [a.arg for a in args.args]: + total_params -= 1 + + if total_params > max_params: + violations.append( + Violation( + rule="long_parameter_list", + relative_path=rel_path, + identifier=node.name, + detail=f"params={total_params}", + ) + ) + + return violations + + +def collect_god_classes( + *, + parse_errors: list[str] | None = None, + max_methods: int = 15, + max_lines: int = 400, +) -> list[Violation]: + """Collect god class violations.""" + violations: list[Violation] = [] + + for py_file in find_source_files(include_migrations=False): + tree = _parse_tree(py_file, parse_errors) + if tree is None: + continue + + rel_path = relative_path(py_file) + + for node in ast.walk(tree): + if isinstance(node, ast.ClassDef): + methods = [ + n + for n in node.body + if isinstance(n, (ast.FunctionDef, ast.AsyncFunctionDef)) + ] + + if len(methods) > max_methods: + violations.append( + Violation( + rule="god_class", + relative_path=rel_path, + identifier=node.name, + detail=f"methods={len(methods)}", + ) + ) + + if node.end_lineno: + class_lines = node.end_lineno - node.lineno + 1 + if class_lines > max_lines: + violations.append( + Violation( + rule="god_class", + relative_path=rel_path, + identifier=node.name, + detail=f"lines={class_lines}", + ) + ) + + return violations + + +def collect_deep_nesting( + *, + parse_errors: list[str] | None = None, + max_nesting: int = 2, +) -> list[Violation]: + """Collect deep nesting violations.""" + violations: list[Violation] = [] + + for py_file in find_source_files(include_migrations=False): + tree = _parse_tree(py_file, parse_errors) + if tree is None: + continue + + rel_path = relative_path(py_file) + + for node in ast.walk(tree): + if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)): + depth = count_nesting_depth(node) + if depth > max_nesting: + violations.append( + Violation( + rule="deep_nesting", + relative_path=rel_path, + identifier=node.name, + detail=f"depth={depth}", + ) + ) + + return violations + + +def collect_long_methods( + *, + parse_errors: list[str] | None = None, + max_lines: int = 50, +) -> list[Violation]: + """Collect long method violations.""" + violations: list[Violation] = [] + + for py_file in find_source_files(include_migrations=False): + tree = _parse_tree(py_file, parse_errors) + if tree is None: + continue + + rel_path = relative_path(py_file) + + for node in ast.walk(tree): + if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)): + lines = count_function_lines(node) + if lines > max_lines: + violations.append( + Violation( + rule="long_method", + relative_path=rel_path, + identifier=node.name, + detail=f"lines={lines}", + ) + ) + + return violations + + +def collect_feature_envy( + *, + parse_errors: list[str] | None = None, +) -> list[Violation]: + """Collect feature envy violations.""" + excluded_class_patterns = { + "converter", + "exporter", + "repository", + } + excluded_method_patterns = { + "_to_domain", + "_to_proto", + "_proto_to_", + "_to_orm", + "_from_orm", + } + excluded_object_names = { + "model", + "meeting", + "segment", + "request", + "response", + "np", + "noteflow_pb2", + "seg", + "job", + "repo", + "ai", + "summary", + "MeetingState", + "event", + "item", + "entity", + "uow", + "span", + "host", + } + + violations: list[Violation] = [] + + for py_file in find_source_files(include_migrations=False): + tree = _parse_tree(py_file, parse_errors) + if tree is None: + continue + + rel_path = relative_path(py_file) + + for class_node in ast.walk(tree): + if not isinstance(class_node, ast.ClassDef): + continue + + if any(p in class_node.name.lower() for p in excluded_class_patterns): + continue + + for method in class_node.body: + if not isinstance(method, (ast.FunctionDef, ast.AsyncFunctionDef)): + continue + + if any(p in method.name.lower() for p in excluded_method_patterns): + continue + + self_accesses = 0 + other_accesses: dict[str, int] = {} + + for node in ast.walk(method): + if isinstance(node, ast.Attribute) and isinstance(node.value, ast.Name): + if node.value.id == "self": + self_accesses += 1 + else: + other_accesses[node.value.id] = other_accesses.get( + node.value.id, 0 + ) + 1 + + for other_obj, count in other_accesses.items(): + if other_obj in excluded_object_names: + continue + if count > self_accesses + 2 and count > 4: + violations.append( + Violation( + rule="feature_envy", + relative_path=rel_path, + identifier=f"{class_node.name}.{method.name}", + detail=f"{other_obj}={count}_vs_self={self_accesses}", + ) + ) + + return violations + + +def collect_module_size_soft( + *, + soft_limit: int = 350, +) -> list[Violation]: + """Collect module size soft limit violations.""" + violations: list[Violation] = [] + + for py_file in find_source_files(include_migrations=False): + content, error = read_file_safe(py_file) + if error or content is None: + continue + + rel_path = relative_path(py_file) + line_count = len(content.splitlines()) + + if line_count > soft_limit: + violations.append( + Violation( + rule="module_size_soft", + relative_path=rel_path, + identifier="module", + detail=f"lines={line_count}", + ) + ) + + return violations diff --git a/tests/quality/_detectors/stale_code.py b/tests/quality/_detectors/stale_code.py new file mode 100644 index 0000000..8c4a443 --- /dev/null +++ b/tests/quality/_detectors/stale_code.py @@ -0,0 +1,163 @@ +"""Shared detectors for stale code quality rules.""" + +from __future__ import annotations + +import ast +import re +from pathlib import Path + +from tests.quality._baseline import Violation, content_hash +from tests.quality._helpers import ( + find_source_files, + parse_file_safe, + read_file_safe, + relative_path, +) + + +def _parse_tree(py_file: Path, parse_errors: list[str] | None) -> ast.AST | None: + tree, error = parse_file_safe(py_file) + if error is not None: + if parse_errors is not None: + parse_errors.append(error) + return None + return tree + + +def collect_stale_todos() -> list[Violation]: + """Collect stale TODO violations.""" + stale_pattern = re.compile( + r"#\s*(TODO|FIXME|HACK|XXX|DEPRECATED)[\s:]*(.{0,100})", + re.IGNORECASE, + ) + violations: list[Violation] = [] + + for py_file in find_source_files(): + content, error = read_file_safe(py_file) + if error or content is None: + continue + + rel_path = relative_path(py_file) + lines = content.splitlines() + + for i, line in enumerate(lines, start=1): + if match := stale_pattern.search(line): + tag = match[1].upper() + message = match[2].strip()[:50] + violations.append( + Violation( + rule="stale_todo", + relative_path=rel_path, + identifier=content_hash(f"{i}:{line.strip()}"), + detail=f"{tag}:{message}", + ) + ) + + return violations + + +def collect_orphaned_imports( + *, + parse_errors: list[str] | None = None, +) -> list[Violation]: + """Collect orphaned import violations.""" + violations: list[Violation] = [] + + for py_file in find_source_files(include_init=False): + tree = _parse_tree(py_file, parse_errors) + if tree is None: + continue + + rel_path = relative_path(py_file) + + imported_names: set[str] = set() + for node in ast.walk(tree): + if isinstance(node, ast.Import): + for alias in node.names: + name = alias.asname or alias.name.split(".")[0] + imported_names.add(name) + elif isinstance(node, ast.ImportFrom): + for alias in node.names: + if alias.name != "*": + name = alias.asname or alias.name + imported_names.add(name) + + used_names: set[str] = set() + for node in ast.walk(tree): + if isinstance(node, ast.Name): + used_names.add(node.id) + elif isinstance(node, ast.Attribute) and isinstance(node.value, ast.Name): + used_names.add(node.value.id) + + all_exports: set[str] = set() + for node in ast.walk(tree): + if isinstance(node, ast.Assign): + for target in node.targets: + if isinstance(target, ast.Name) and target.id == "__all__": + if isinstance(node.value, ast.List): + for elt in node.value.elts: + if isinstance(elt, ast.Constant) and isinstance( + elt.value, str + ): + all_exports.add(elt.value) + + type_checking_imports: set[str] = set() + for node in ast.walk(tree): + if isinstance(node, ast.If): + if isinstance(node.test, ast.Name) and node.test.id == "TYPE_CHECKING": + for subnode in ast.walk(node): + if isinstance(subnode, ast.ImportFrom): + for alias in subnode.names: + name = alias.asname or alias.name + type_checking_imports.add(name) + + unused = imported_names - used_names - type_checking_imports - all_exports + unused -= {"__future__", "annotations"} + + violations.extend( + Violation( + rule="orphaned_import", + relative_path=rel_path, + identifier=name, + ) + for name in sorted(unused) + if not name.startswith("_") + ) + return violations + + +def collect_deprecated_patterns() -> list[Violation]: + """Collect deprecated pattern violations.""" + deprecated_patterns = [ + (r"\.format\s*\(", "str.format()"), + (r"% \(", "%-formatting"), + (r"from typing import Optional", "Optional[X]"), + (r"from typing import Union", "Union[X, Y]"), + (r"from typing import List\b", "List[X]"), + (r"from typing import Dict\b", "Dict[K, V]"), + (r"from typing import Tuple\b", "Tuple[X, ...]"), + (r"from typing import Set\b", "Set[X]"), + ] + + violations: list[Violation] = [] + + for py_file in find_source_files(): + content, error = read_file_safe(py_file) + if error or content is None: + continue + + rel_path = relative_path(py_file) + lines = content.splitlines() + + for i, line in enumerate(lines, start=1): + violations.extend( + Violation( + rule="deprecated_pattern", + relative_path=rel_path, + identifier=content_hash(f"{i}:{line.strip()}"), + detail=old_style, + ) + for pattern, old_style in deprecated_patterns + if re.search(pattern, line) + ) + return violations diff --git a/tests/quality/_test_smell_collectors.py b/tests/quality/_detectors/test_smells.py similarity index 76% rename from tests/quality/_test_smell_collectors.py rename to tests/quality/_detectors/test_smells.py index d7d66d8..1982c11 100644 --- a/tests/quality/_test_smell_collectors.py +++ b/tests/quality/_detectors/test_smells.py @@ -1,8 +1,4 @@ -"""Test smell violation collectors for baseline generation. - -These collectors are used by generate_baseline.py to capture current -test smell violations for baseline enforcement. -""" +"""Shared detectors for test smell quality rules.""" from __future__ import annotations @@ -20,7 +16,16 @@ from tests.quality._helpers import ( ) -def _get_test_methods(tree: ast.AST) -> list[ast.FunctionDef | ast.AsyncFunctionDef]: +def _parse_tree(py_file: Path, parse_errors: list[str] | None) -> ast.AST | None: + tree, error = parse_file_safe(py_file) + if error is not None: + if parse_errors is not None: + parse_errors.append(error) + return None + return tree + + +def get_test_methods(tree: ast.AST) -> list[ast.FunctionDef | ast.AsyncFunctionDef]: """Extract test methods from AST.""" tests: list[ast.FunctionDef | ast.AsyncFunctionDef] = [] for node in ast.walk(tree): @@ -30,7 +35,7 @@ def _get_test_methods(tree: ast.AST) -> list[ast.FunctionDef | ast.AsyncFunction return tests -def _get_fixtures(tree: ast.AST) -> list[ast.FunctionDef]: +def get_fixtures(tree: ast.AST) -> list[ast.FunctionDef]: """Extract pytest fixtures from AST.""" fixtures: list[ast.FunctionDef] = [] for node in ast.walk(tree): @@ -51,7 +56,7 @@ def _get_fixtures(tree: ast.AST) -> list[ast.FunctionDef]: return fixtures -def _get_fixture_scope(node: ast.FunctionDef) -> str | None: +def get_fixture_scope(node: ast.FunctionDef) -> str | None: """Extract fixture scope from decorator.""" for decorator in node.decorator_list: if isinstance(decorator, ast.Call): @@ -62,18 +67,45 @@ def _get_fixture_scope(node: ast.FunctionDef) -> str | None: return None -def collect_assertion_roulette() -> list[Violation]: +def get_module_level_fixtures(tree: ast.AST) -> list[ast.FunctionDef]: + """Extract only module-level pytest fixtures from AST (not class-scoped).""" + fixtures: list[ast.FunctionDef] = [] + if not isinstance(tree, ast.Module): + return fixtures + for node in tree.body: + if isinstance(node, ast.FunctionDef): + for decorator in node.decorator_list: + if isinstance(decorator, ast.Attribute): + if decorator.attr == "fixture": + fixtures.append(node) + break + elif isinstance(decorator, ast.Call): + if isinstance(decorator.func, ast.Attribute): + if decorator.func.attr == "fixture": + fixtures.append(node) + break + elif isinstance(decorator, ast.Name) and decorator.id == "fixture": + fixtures.append(node) + break + return fixtures + + +def _contains_assertion(node: ast.AST) -> bool: + return any(isinstance(child, ast.Assert) for child in ast.walk(node)) + + +def collect_assertion_roulette(*, parse_errors: list[str] | None = None) -> list[Violation]: """Collect assertion roulette violations.""" violations: list[Violation] = [] for py_file in find_test_files(): - tree, error = parse_file_safe(py_file) - if error or tree is None: + tree = _parse_tree(py_file, parse_errors) + if tree is None: continue rel_path = relative_path(py_file) - for test_method in _get_test_methods(tree): + for test_method in get_test_methods(tree): assertions_without_msg = 0 for node in ast.walk(test_method): if isinstance(node, ast.Assert): @@ -93,24 +125,24 @@ def collect_assertion_roulette() -> list[Violation]: return violations -def collect_conditional_test_logic() -> list[Violation]: +def collect_conditional_test_logic( + *, + parse_errors: list[str] | None = None, +) -> list[Violation]: """Collect conditional test logic violations.""" violations: list[Violation] = [] - def _contains_assertion(node: ast.AST) -> bool: - return any(isinstance(child, ast.Assert) for child in ast.walk(node)) - for py_file in find_test_files(): if "stress" in py_file.parts: continue - tree, error = parse_file_safe(py_file) - if error or tree is None: + tree = _parse_tree(py_file, parse_errors) + if tree is None: continue rel_path = relative_path(py_file) - for test_method in _get_test_methods(tree): + for test_method in get_test_methods(tree): conditionals: list[str] = [] for node in ast.walk(test_method): @@ -134,7 +166,7 @@ def collect_conditional_test_logic() -> list[Violation]: return violations -def collect_sleepy_tests() -> list[Violation]: +def collect_sleepy_tests(*, parse_errors: list[str] | None = None) -> list[Violation]: """Collect sleepy test violations.""" allowed_sleepy_paths = { "tests/stress/", @@ -146,11 +178,11 @@ def collect_sleepy_tests() -> list[Violation]: if any(allowed in rel_path for allowed in allowed_sleepy_paths): continue - tree, error = parse_file_safe(py_file) - if error or tree is None: + tree = _parse_tree(py_file, parse_errors) + if tree is None: continue - for test_method in _get_test_methods(tree): + for test_method in get_test_methods(tree): for node in ast.walk(test_method): if isinstance(node, ast.Call): if isinstance(node.func, ast.Attribute): @@ -169,18 +201,18 @@ def collect_sleepy_tests() -> list[Violation]: return violations -def collect_unknown_tests() -> list[Violation]: +def collect_unknown_tests(*, parse_errors: list[str] | None = None) -> list[Violation]: """Collect unknown test (no assertion) violations.""" violations: list[Violation] = [] for py_file in find_test_files(): - tree, error = parse_file_safe(py_file) - if error or tree is None: + tree = _parse_tree(py_file, parse_errors) + if tree is None: continue rel_path = relative_path(py_file) - for test_method in _get_test_methods(tree): + for test_method in get_test_methods(tree): has_assertion = False has_raises = False @@ -210,18 +242,21 @@ def collect_unknown_tests() -> list[Violation]: return violations -def collect_redundant_prints() -> list[Violation]: +def collect_redundant_prints( + *, + parse_errors: list[str] | None = None, +) -> list[Violation]: """Collect redundant print violations in tests.""" violations: list[Violation] = [] for py_file in find_test_files(): - tree, error = parse_file_safe(py_file) - if error or tree is None: + tree = _parse_tree(py_file, parse_errors) + if tree is None: continue rel_path = relative_path(py_file) - for test_method in _get_test_methods(tree): + for test_method in get_test_methods(tree): for node in ast.walk(test_method): if isinstance(node, ast.Call) and isinstance(node.func, ast.Name): if node.func.id == "print": @@ -237,18 +272,21 @@ def collect_redundant_prints() -> list[Violation]: return violations -def collect_exception_handling() -> list[Violation]: +def collect_exception_handling( + *, + parse_errors: list[str] | None = None, +) -> list[Violation]: """Collect exception handling in tests violations.""" violations: list[Violation] = [] for py_file in find_test_files(): - tree, error = parse_file_safe(py_file) - if error or tree is None: + tree = _parse_tree(py_file, parse_errors) + if tree is None: continue rel_path = relative_path(py_file) - for test_method in _get_test_methods(tree): + for test_method in get_test_methods(tree): for node in ast.walk(test_method): if isinstance(node, ast.Try): for handler in node.handlers: @@ -275,19 +313,22 @@ def collect_exception_handling() -> list[Violation]: return violations -def collect_magic_number_tests() -> list[Violation]: +def collect_magic_number_tests( + *, + parse_errors: list[str] | None = None, +) -> list[Violation]: """Collect magic number in test assertions violations.""" allowed_numbers = {0, 1, 2, -1, 100, 1000} violations: list[Violation] = [] for py_file in find_test_files(): - tree, error = parse_file_safe(py_file) - if error or tree is None: + tree = _parse_tree(py_file, parse_errors) + if tree is None: continue rel_path = relative_path(py_file) - for test_method in _get_test_methods(tree): + for test_method in get_test_methods(tree): for node in ast.walk(test_method): if isinstance(node, ast.Assert): for child in ast.walk(node): @@ -307,7 +348,10 @@ def collect_magic_number_tests() -> list[Violation]: return violations -def collect_sensitive_equality() -> list[Violation]: +def collect_sensitive_equality( + *, + parse_errors: list[str] | None = None, +) -> list[Violation]: """Collect sensitive equality (str/repr comparison) violations.""" excluded_test_patterns = {"string", "proto"} excluded_file_patterns: set[str] = set() @@ -317,13 +361,13 @@ def collect_sensitive_equality() -> list[Violation]: if any(p in py_file.stem.lower() for p in excluded_file_patterns): continue - tree, error = parse_file_safe(py_file) - if error or tree is None: + tree = _parse_tree(py_file, parse_errors) + if tree is None: continue rel_path = relative_path(py_file) - for test_method in _get_test_methods(tree): + for test_method in get_test_methods(tree): if any(p in test_method.name.lower() for p in excluded_test_patterns): continue for node in ast.walk(test_method): @@ -346,19 +390,19 @@ def collect_sensitive_equality() -> list[Violation]: return violations -def collect_eager_tests() -> list[Violation]: +def collect_eager_tests(*, parse_errors: list[str] | None = None) -> list[Violation]: """Collect eager test (too many method calls) violations.""" max_method_calls = 7 violations: list[Violation] = [] for py_file in find_test_files(): - tree, error = parse_file_safe(py_file) - if error or tree is None: + tree = _parse_tree(py_file, parse_errors) + if tree is None: continue rel_path = relative_path(py_file) - for test_method in _get_test_methods(tree): + for test_method in get_test_methods(tree): method_calls: set[str] = set() for node in ast.walk(test_method): @@ -380,16 +424,19 @@ def collect_eager_tests() -> list[Violation]: return violations -def collect_duplicate_test_names() -> list[Violation]: +def collect_duplicate_test_names( + *, + parse_errors: list[str] | None = None, +) -> list[Violation]: """Collect duplicate test name violations.""" test_names: dict[str, list[tuple[Path, int]]] = defaultdict(list) for py_file in find_test_files(): - tree, error = parse_file_safe(py_file) - if error or tree is None: + tree = _parse_tree(py_file, parse_errors) + if tree is None: continue - for test_method in _get_test_methods(tree): + for test_method in get_test_methods(tree): test_names[test_method.name].append((py_file, test_method.lineno)) duplicates = {name: locs for name, locs in test_names.items() if len(locs) > 1} @@ -409,19 +456,19 @@ def collect_duplicate_test_names() -> list[Violation]: return violations -def collect_long_tests() -> list[Violation]: +def collect_long_tests(*, parse_errors: list[str] | None = None) -> list[Violation]: """Collect long test method violations.""" max_lines = 35 violations: list[Violation] = [] for py_file in find_test_files(): - tree, error = parse_file_safe(py_file) - if error or tree is None: + tree = _parse_tree(py_file, parse_errors) + if tree is None: continue rel_path = relative_path(py_file) - for test_method in _get_test_methods(tree): + for test_method in get_test_methods(tree): if test_method.end_lineno: lines = test_method.end_lineno - test_method.lineno + 1 if lines > max_lines: @@ -437,18 +484,21 @@ def collect_long_tests() -> list[Violation]: return violations -def collect_fixture_missing_type() -> list[Violation]: +def collect_fixture_missing_type( + *, + parse_errors: list[str] | None = None, +) -> list[Violation]: """Collect fixtures missing type hints violations.""" violations: list[Violation] = [] for py_file in find_test_files(): - tree, error = parse_file_safe(py_file) - if error or tree is None: + tree = _parse_tree(py_file, parse_errors) + if tree is None: continue rel_path = relative_path(py_file) - for fixture in _get_fixtures(tree): + for fixture in get_fixtures(tree): if fixture.returns is None: if not fixture.name.startswith("_"): violations.append( @@ -462,18 +512,21 @@ def collect_fixture_missing_type() -> list[Violation]: return violations -def collect_unused_fixtures() -> list[Violation]: +def collect_unused_fixtures( + *, + parse_errors: list[str] | None = None, +) -> list[Violation]: """Collect unused fixture parameter violations.""" violations: list[Violation] = [] for py_file in find_test_files(): - tree, error = parse_file_safe(py_file) - if error or tree is None: + tree = _parse_tree(py_file, parse_errors) + if tree is None: continue rel_path = relative_path(py_file) - for test_method in _get_test_methods(tree): + for test_method in get_test_methods(tree): params = [ arg.arg for arg in test_method.args.args if arg.arg not in ("self", "cls") @@ -486,9 +539,18 @@ def collect_unused_fixtures() -> list[Violation]: if isinstance(node, ast.Name) } skip_params = { - "monkeypatch", "capsys", "capfd", "caplog", "tmp_path", - "tmp_path_factory", "request", "pytestconfig", "record_property", - "record_testsuite_property", "recwarn", "event_loop", + "monkeypatch", + "capsys", + "capfd", + "caplog", + "tmp_path", + "tmp_path_factory", + "request", + "pytestconfig", + "record_property", + "record_testsuite_property", + "recwarn", + "event_loop", } violations.extend( @@ -504,7 +566,10 @@ def collect_unused_fixtures() -> list[Violation]: return violations -def collect_fixture_scope_too_narrow() -> list[Violation]: +def collect_fixture_scope_too_narrow( + *, + parse_errors: list[str] | None = None, +) -> list[Violation]: """Collect fixtures with potentially wrong scope violations.""" expensive_patterns = [ r"asyncpg\.connect", @@ -518,8 +583,8 @@ def collect_fixture_scope_too_narrow() -> list[Violation]: violations: list[Violation] = [] for py_file in find_test_files(): - tree, error = parse_file_safe(py_file) - if error or tree is None: + tree = _parse_tree(py_file, parse_errors) + if tree is None: continue content, _ = read_file_safe(py_file) @@ -528,8 +593,8 @@ def collect_fixture_scope_too_narrow() -> list[Violation]: rel_path = relative_path(py_file) - for fixture in _get_fixtures(tree): - scope = _get_fixture_scope(fixture) + for fixture in get_fixtures(tree): + scope = get_fixture_scope(fixture) if fixture_source := ast.get_source_segment(content, fixture): if scope is None or scope == "function": for pattern in expensive_patterns: @@ -547,18 +612,21 @@ def collect_fixture_scope_too_narrow() -> list[Violation]: return violations -def collect_raises_without_match() -> list[Violation]: +def collect_raises_without_match( + *, + parse_errors: list[str] | None = None, +) -> list[Violation]: """Collect pytest.raises without match violations.""" violations: list[Violation] = [] for py_file in find_test_files(): - tree, error = parse_file_safe(py_file) - if error or tree is None: + tree = _parse_tree(py_file, parse_errors) + if tree is None: continue rel_path = relative_path(py_file) - for test_method in _get_test_methods(tree): + for test_method in get_test_methods(tree): for node in ast.walk(test_method): if isinstance(node, ast.Call): if isinstance(node.func, ast.Attribute): diff --git a/tests/quality/_detectors/wrappers.py b/tests/quality/_detectors/wrappers.py new file mode 100644 index 0000000..c949e41 --- /dev/null +++ b/tests/quality/_detectors/wrappers.py @@ -0,0 +1,230 @@ +"""Shared detectors for unnecessary wrappers and aliases.""" + +from __future__ import annotations + +import ast +import re +from pathlib import Path + +from tests.quality._baseline import Violation, content_hash +from tests.quality._helpers import ( + find_source_files, + parse_file_safe, + read_file_safe, + relative_path, +) + +ALLOWED_WRAPPERS: set[tuple[str, str]] = { + ("get_settings", "_load_settings"), + ("get_trigger_settings", "_load_trigger_settings"), + ("get_feature_flags", "model_validate"), + ("get_calendar_settings", "model_validate"), + ("from_args", "cls"), + ("segment_count", "len"), + ("full_transcript", "join"), + ("duration", "sub"), + ("is_active", "property"), + ("get_metadata", "get"), + ("evaluate", "RuleResult"), + ("database_url_str", "str"), + ("generate_request_id", "str"), + ("get_request_id", "get"), + ("datetime_to_epoch_seconds", "timestamp"), + ("datetime_to_iso_string", "isoformat"), + ("from_metrics", "cls"), + ("from_dict", "cls"), + ("pending", "cls"), + ("running", "cls"), + ("completed", "cls"), + ("get_log_level", "get"), + ("get_provider", "get"), + ("process_chunk", "process"), + ("get_openai_client", "_get_openai_client"), + ("meeting_apps", "frozenset"), + ("get_sync_run", "get"), + ("list_all", "list"), + ("get_by_id", "get"), + ("check_otel_available", "_check_otel_available"), + ("start_span", "_NoOpSpan"), + ("detected_app", "next"), +} + + +def _parse_tree(py_file: Path, parse_errors: list[str] | None) -> ast.AST | None: + tree, error = parse_file_safe(py_file) + if error is not None: + if parse_errors is not None: + parse_errors.append(error) + return None + return tree + + +def is_thin_wrapper(node: ast.FunctionDef | ast.AsyncFunctionDef) -> str | None: + """Check if function is a thin wrapper returning another call directly.""" + body_stmts = [ + s + for s in node.body + if not isinstance(s, ast.Expr) or not isinstance(s.value, ast.Constant) + ] + + if len(body_stmts) == 1: + stmt = body_stmts[0] + if isinstance(stmt, ast.Return) and isinstance(stmt.value, ast.Call): + call = stmt.value + if isinstance(call.func, ast.Name): + return call.func.id + if isinstance(call.func, ast.Attribute): + return call.func.attr + return None + + +def collect_thin_wrappers( + *, + parse_errors: list[str] | None = None, +) -> list[Violation]: + """Collect thin wrapper violations.""" + violations: list[Violation] = [] + + for py_file in find_source_files(): + tree = _parse_tree(py_file, parse_errors) + if tree is None: + continue + + rel_path = relative_path(py_file) + if "converters" in rel_path: + continue + + for node in ast.walk(tree): + if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)): + if node.name.startswith("_"): + continue + + wrapped = is_thin_wrapper(node) + if wrapped and node.name != wrapped: + if (node.name, wrapped) in ALLOWED_WRAPPERS: + continue + violations.append( + Violation( + rule="thin_wrapper", + relative_path=rel_path, + identifier=node.name, + detail=wrapped, + ) + ) + + return violations + + +def collect_alias_imports() -> list[Violation]: + """Collect alias import violations.""" + alias_pattern = re.compile(r"^import\s+(\w+)\s+as\s+(\w+)") + from_alias_pattern = re.compile(r"from\s+\S+\s+import\s+(\w+)\s+as\s+(\w+)") + + violations: list[Violation] = [] + + for py_file in find_source_files(): + content, error = read_file_safe(py_file) + if error or content is None: + continue + + rel_path = relative_path(py_file) + lines = content.splitlines() + + for i, line in enumerate(lines, start=1): + for pattern in [alias_pattern, from_alias_pattern]: + if match := pattern.search(line): + original, alias = match.groups() + if original.lower() not in alias.lower(): + if alias not in {"np", "pd", "plt", "tf", "nn", "F", "sa", "sd"}: + violations.append( + Violation( + rule="alias_import", + relative_path=rel_path, + identifier=content_hash(f"{i}:{line.strip()}"), + detail=f"{original}->{alias}", + ) + ) + + return violations + + +def collect_redundant_type_aliases( + *, + parse_errors: list[str] | None = None, +) -> list[Violation]: + """Collect redundant type alias violations.""" + violations: list[Violation] = [] + + for py_file in find_source_files(): + tree = _parse_tree(py_file, parse_errors) + if tree is None: + continue + + rel_path = relative_path(py_file) + + for node in ast.walk(tree): + if isinstance(node, ast.AnnAssign) and isinstance(node.target, ast.Name): + if isinstance(node.annotation, ast.Name): + if node.annotation.id == "TypeAlias": + if isinstance(node.value, ast.Name): + base_type = node.value.id + if base_type in {"str", "int", "float", "bool", "bytes"}: + target_name = node.target.id + violations.append( + Violation( + rule="redundant_type_alias", + relative_path=rel_path, + identifier=target_name, + detail=base_type, + ) + ) + + return violations + + +def collect_passthrough_classes( + *, + parse_errors: list[str] | None = None, + allowed_factory_classes: set[str] | None = None, +) -> list[Violation]: + """Collect passthrough class violations.""" + if allowed_factory_classes is None: + allowed_factory_classes = set() + + violations: list[Violation] = [] + + for py_file in find_source_files(): + tree = _parse_tree(py_file, parse_errors) + if tree is None: + continue + + rel_path = relative_path(py_file) + + for node in ast.walk(tree): + if isinstance(node, ast.ClassDef): + if node.name in allowed_factory_classes: + continue + + methods = [ + n + for n in node.body + if isinstance(n, (ast.FunctionDef, ast.AsyncFunctionDef)) + and not n.name.startswith("_") + ] + + if len(methods) >= 3: + passthrough_count = sum( + 1 for m in methods if is_thin_wrapper(m) is not None + ) + + if passthrough_count == len(methods): + violations.append( + Violation( + rule="passthrough_class", + relative_path=rel_path, + identifier=node.name, + detail=f"{passthrough_count}_methods", + ) + ) + + return violations diff --git a/tests/quality/baselines.json b/tests/quality/baselines.json index 19f588d..22f2845 100644 --- a/tests/quality/baselines.json +++ b/tests/quality/baselines.json @@ -1,23 +1,45 @@ { - "generated_at": "2026-01-14T04:30:39.258014+00:00", + "generated_at": "2026-01-15T00:00:00.000000+00:00", "rules": { "deep_nesting": [ - "deep_nesting|src/noteflow/application/services/asr_config_service.py|_execute_reconfiguration|depth=3", - "deep_nesting|src/noteflow/application/services/asr_config_service.py|shutdown|depth=3", - "deep_nesting|src/noteflow/grpc/_mixins/diarization/_jobs.py|_execute_diarization|depth=3", - "deep_nesting|src/noteflow/grpc/_mixins/sync.py|perform_sync|depth=3", - "deep_nesting|src/noteflow/grpc/_service_shutdown.py|cancel_sync_tasks|depth=3" + "deep_nesting|src/noteflow/application/services/asr_config/service.py|_execute_reconfiguration|depth=3", + "deep_nesting|src/noteflow/application/services/asr_config/service.py|shutdown|depth=3", + "deep_nesting|src/noteflow/grpc/mixins/diarization/_jobs.py|_execute_diarization|depth=3", + "deep_nesting|src/noteflow/grpc/mixins/sync.py|perform_sync|depth=3", + "deep_nesting|src/noteflow/grpc/servicer/shutdown.py|cancel_sync_tasks|depth=3" + ], + "feature_envy": [ + "feature_envy|src/noteflow/application/services/recovery/recovery_service.py|RecoveryService.recover_all|job_result=5_vs_self=2" ], "god_class": [ - "god_class|src/noteflow/application/services/asr_config_service.py|AsrConfigService|methods=16" + "god_class|src/noteflow/application/services/asr_config/service.py|AsrConfigService|methods=16", + "god_class|src/noteflow/grpc/client_mixins/protocols.py|ClientHost|methods=23" ], "long_method": [ - "long_method|src/noteflow/grpc/_mixins/sync.py|perform_sync|lines=60" + "long_method|src/noteflow/application/services/auth/workflows.py|refresh_tokens_for_integration|lines=55", + "long_method|src/noteflow/grpc/mixins/_task_callbacks.py|create_job_done_callback|lines=60", + "long_method|src/noteflow/grpc/mixins/diarization/_jobs.py|start_diarization_job|lines=52", + "long_method|src/noteflow/grpc/mixins/errors/_webhooks.py|fire_webhook_safe|lines=54" + ], + "long_parameter_list": [ + "long_parameter_list|src/noteflow/grpc/client_mixins/annotation.py|add_annotation_result|params=6", + "long_parameter_list|src/noteflow/grpc/client_mixins/annotation.py|update_annotation_result|params=5", + "long_parameter_list|src/noteflow/grpc/client_mixins/protocols.py|add_annotation_result|params=6", + "long_parameter_list|src/noteflow/grpc/client_mixins/protocols.py|update_annotation_result|params=5", + "long_parameter_list|src/noteflow/grpc/mixins/streaming/_processing/_audio_ops.py|decode_and_convert_audio|params=5" ], "module_size_soft": [ - "module_size_soft|src/noteflow/application/services/asr_config_service.py|module|lines=361", - "module_size_soft|src/noteflow/grpc/_mixins/diarization/_jobs.py|module|lines=372", - "module_size_soft|src/noteflow/grpc/_mixins/sync.py|module|lines=384" + "module_size_soft|src/noteflow/grpc/mixins/diarization/_jobs.py|module|lines=372" + ], + "thin_wrapper": [ + "thin_wrapper|src/noteflow/grpc/types/__init__.py|Ok|ClientResult", + "thin_wrapper|src/noteflow/grpc/types/__init__.py|Err|ClientResult", + "thin_wrapper|src/noteflow/infrastructure/persistence/models/_columns.py|jsonb_dict_column|mapped_column", + "thin_wrapper|src/noteflow/infrastructure/persistence/models/_columns.py|metadata_column|mapped_column", + "thin_wrapper|src/noteflow/infrastructure/persistence/models/_columns.py|utc_now_column|mapped_column", + "thin_wrapper|src/noteflow/infrastructure/persistence/models/_columns.py|utc_now_onupdate_column|mapped_column", + "thin_wrapper|src/noteflow/infrastructure/persistence/models/_columns.py|meeting_id_fk_column|mapped_column", + "thin_wrapper|src/noteflow/infrastructure/persistence/models/_columns.py|workspace_id_fk_column|mapped_column" ] }, "schema_version": 1 diff --git a/tests/quality/generate_baseline.py b/tests/quality/generate_baseline.py index 294f901..ba65efe 100644 --- a/tests/quality/generate_baseline.py +++ b/tests/quality/generate_baseline.py @@ -12,8 +12,6 @@ Usage: from __future__ import annotations -import ast -import re import sys from pathlib import Path @@ -22,16 +20,23 @@ sys.path.insert(0, str(Path(__file__).parent.parent.parent)) from tests.quality._baseline import ( Violation, - content_hash, save_baseline, ) -from tests.quality._helpers import ( - find_source_files, - parse_file_safe, - read_file_safe, - relative_path, +from tests.quality._detectors.code_smells import ( + collect_deep_nesting, + collect_feature_envy, + collect_god_classes, + collect_high_complexity, + collect_long_methods, + collect_long_parameter_lists, + collect_module_size_soft, ) -from tests.quality._test_smell_collectors import ( +from tests.quality._detectors.stale_code import ( + collect_deprecated_patterns, + collect_orphaned_imports, + collect_stale_todos, +) +from tests.quality._detectors.test_smells import ( collect_assertion_roulette, collect_conditional_test_logic, collect_duplicate_test_names, @@ -48,660 +53,12 @@ from tests.quality._test_smell_collectors import ( collect_unknown_tests, collect_unused_fixtures, ) - - -def collect_stale_todos() -> list[Violation]: - """Collect stale TODO violations.""" - stale_pattern = re.compile( - r"#\s*(TODO|FIXME|HACK|XXX|DEPRECATED)[\s:]*(.{0,100})", - re.IGNORECASE, - ) - violations: list[Violation] = [] - - for py_file in find_source_files(): - content, error = read_file_safe(py_file) - if error or content is None: - continue - - rel_path = relative_path(py_file) - lines = content.splitlines() - - for i, line in enumerate(lines, start=1): - if match := stale_pattern.search(line): - tag = match[1].upper() - message = match[2].strip()[:50] - violations.append( - Violation( - rule="stale_todo", - relative_path=rel_path, - identifier=content_hash(f"{i}:{line.strip()}"), - detail=f"{tag}:{message}", - ) - ) - - return violations - - -def collect_orphaned_imports() -> list[Violation]: - """Collect orphaned import violations.""" - violations: list[Violation] = [] - - for py_file in find_source_files(include_init=False): - tree, error = parse_file_safe(py_file) - if error or tree is None: - continue - - rel_path = relative_path(py_file) - - imported_names: set[str] = set() - for node in ast.walk(tree): - if isinstance(node, ast.Import): - for alias in node.names: - name = alias.asname or alias.name.split(".")[0] - imported_names.add(name) - elif isinstance(node, ast.ImportFrom): - for alias in node.names: - if alias.name != "*": - name = alias.asname or alias.name - imported_names.add(name) - - used_names: set[str] = set() - for node in ast.walk(tree): - if isinstance(node, ast.Name): - used_names.add(node.id) - elif isinstance(node, ast.Attribute) and isinstance(node.value, ast.Name): - used_names.add(node.value.id) - - # Check for __all__ re-exports - all_exports: set[str] = set() - for node in ast.walk(tree): - if isinstance(node, ast.Assign): - for target in node.targets: - if isinstance(target, ast.Name) and target.id == "__all__": - if isinstance(node.value, ast.List): - for elt in node.value.elts: - if isinstance(elt, ast.Constant) and isinstance( - elt.value, str - ): - all_exports.add(elt.value) - - # Check for TYPE_CHECKING imports - type_checking_imports: set[str] = set() - for node in ast.walk(tree): - if isinstance(node, ast.If): - if isinstance(node.test, ast.Name) and node.test.id == "TYPE_CHECKING": - for subnode in ast.walk(node): - if isinstance(subnode, ast.ImportFrom): - for alias in subnode.names: - name = alias.asname or alias.name - type_checking_imports.add(name) - - unused = imported_names - used_names - type_checking_imports - all_exports - unused -= {"__future__", "annotations"} - - violations.extend( - Violation( - rule="orphaned_import", - relative_path=rel_path, - identifier=name, - ) - for name in sorted(unused) - if not name.startswith("_") - ) - return violations - - -def collect_deprecated_patterns() -> list[Violation]: - """Collect deprecated pattern violations.""" - deprecated_patterns = [ - (r"\.format\s*\(", "str.format()"), - (r"% \(", "%-formatting"), - (r"from typing import Optional", "Optional[X]"), - (r"from typing import Union", "Union[X, Y]"), - (r"from typing import List\b", "List[X]"), - (r"from typing import Dict\b", "Dict[K, V]"), - (r"from typing import Tuple\b", "Tuple[X, ...]"), - (r"from typing import Set\b", "Set[X]"), - ] - - violations: list[Violation] = [] - - for py_file in find_source_files(): - content, error = read_file_safe(py_file) - if error or content is None: - continue - - rel_path = relative_path(py_file) - lines = content.splitlines() - - for i, line in enumerate(lines, start=1): - violations.extend( - Violation( - rule="deprecated_pattern", - relative_path=rel_path, - identifier=content_hash(f"{i}:{line.strip()}"), - detail=old_style, - ) - for pattern, old_style in deprecated_patterns - if re.search(pattern, line) - ) - return violations - - -def collect_high_complexity() -> list[Violation]: - """Collect high complexity violations.""" - max_complexity = 12 - violations: list[Violation] = [] - - def count_branches(node: ast.AST) -> int: - count = 0 - for child in ast.walk(node): - if isinstance( - child, (ast.If, ast.While, ast.For, ast.AsyncFor, ast.And, ast.Or) - ): - count += 1 - elif isinstance(child, ast.comprehension): - count += 1 - count += len(child.ifs) - elif isinstance(child, ast.ExceptHandler): - count += 1 - return count - - for py_file in find_source_files(include_migrations=False): - tree, error = parse_file_safe(py_file) - if error or tree is None: - continue - - rel_path = relative_path(py_file) - - for node in ast.walk(tree): - if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)): - complexity = count_branches(node) + 1 - if complexity > max_complexity: - violations.append( - Violation( - rule="high_complexity", - relative_path=rel_path, - identifier=node.name, - detail=f"complexity={complexity}", - ) - ) - - return violations - - -def collect_long_parameter_lists() -> list[Violation]: - """Collect long parameter list violations.""" - max_params = 4 - violations: list[Violation] = [] - - for py_file in find_source_files(include_migrations=False): - tree, error = parse_file_safe(py_file) - if error or tree is None: - continue - - rel_path = relative_path(py_file) - - for node in ast.walk(tree): - if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)): - args = node.args - total_params = ( - len(args.args) + len(args.posonlyargs) + len(args.kwonlyargs) - ) - if "self" in [a.arg for a in args.args]: - total_params -= 1 - if "cls" in [a.arg for a in args.args]: - total_params -= 1 - - if total_params > max_params: - violations.append( - Violation( - rule="long_parameter_list", - relative_path=rel_path, - identifier=node.name, - detail=f"params={total_params}", - ) - ) - - return violations - - -def collect_thin_wrappers() -> list[Violation]: - """Collect thin wrapper violations.""" - allowed_wrappers = { - # Public API facades - ("get_settings", "_load_settings"), - ("get_trigger_settings", "_load_trigger_settings"), - # Cached settings loaders (@lru_cache provides memoization) - ("get_feature_flags", "model_validate"), - ("get_calendar_settings", "model_validate"), - # Factory patterns - ("from_args", "cls"), - # Properties that add semantic meaning - ("segment_count", "len"), - ("full_transcript", "join"), - ("duration", "sub"), - ("is_active", "property"), - # Domain method accessors (type-safe dict access) - ("get_metadata", "get"), - # Strategy pattern implementations (RuleType.evaluate for simple mode) - ("evaluate", "RuleResult"), - # Type conversions - ("database_url_str", "str"), - ("generate_request_id", "str"), - # Context variable accessors (public API over internal contextvars) - ("get_request_id", "get"), - # Time conversion utilities (semantic naming for datetime operations) - ("datetime_to_epoch_seconds", "timestamp"), - ("datetime_to_iso_string", "isoformat"), - # Accessor-style wrappers with semantic names - ("from_metrics", "cls"), - ("from_dict", "cls"), - ("get_log_level", "get"), - ("get_provider", "get"), - ("process_chunk", "process"), - ("get_openai_client", "_get_openai_client"), - ("meeting_apps", "frozenset"), - ("get_sync_run", "get"), - ("list_all", "list"), - ("get_by_id", "get"), - ("check_otel_available", "_check_otel_available"), - ("start_span", "_NoOpSpan"), - ("detected_app", "next"), - } - - def is_thin_wrapper(node: ast.FunctionDef | ast.AsyncFunctionDef) -> str | None: - body_stmts = [ - s - for s in node.body - if not isinstance(s, ast.Expr) or not isinstance(s.value, ast.Constant) - ] - if len(body_stmts) == 1: - stmt = body_stmts[0] - if isinstance(stmt, ast.Return) and isinstance(stmt.value, ast.Call): - call = stmt.value - if isinstance(call.func, ast.Name): - return call.func.id - elif isinstance(call.func, ast.Attribute): - return call.func.attr - return None - - violations: list[Violation] = [] - - for py_file in find_source_files(): - tree, error = parse_file_safe(py_file) - if error or tree is None: - continue - - rel_path = relative_path(py_file) - if "converters" in rel_path: - continue - - for node in ast.walk(tree): - if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)): - if node.name.startswith("_"): - continue - - wrapped = is_thin_wrapper(node) - if wrapped and node.name != wrapped: - if (node.name, wrapped) in allowed_wrappers: - continue - violations.append( - Violation( - rule="thin_wrapper", - relative_path=rel_path, - identifier=node.name, - detail=wrapped, - ) - ) - - return violations - - -def collect_long_methods() -> list[Violation]: - """Collect long method violations.""" - max_lines = 50 - violations: list[Violation] = [] - - def count_function_lines(node: ast.FunctionDef | ast.AsyncFunctionDef) -> int: - return 0 if node.end_lineno is None else node.end_lineno - node.lineno + 1 - - for py_file in find_source_files(include_migrations=False): - tree, error = parse_file_safe(py_file) - if error or tree is None: - continue - - rel_path = relative_path(py_file) - - for node in ast.walk(tree): - if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)): - lines = count_function_lines(node) - if lines > max_lines: - violations.append( - Violation( - rule="long_method", - relative_path=rel_path, - identifier=node.name, - detail=f"lines={lines}", - ) - ) - - return violations - - -def collect_module_size_soft() -> list[Violation]: - """Collect module size soft limit violations.""" - soft_limit = 350 - violations: list[Violation] = [] - - for py_file in find_source_files(include_migrations=False): - content, error = read_file_safe(py_file) - if error or content is None: - continue - - rel_path = relative_path(py_file) - line_count = len(content.splitlines()) - - if line_count > soft_limit: - violations.append( - Violation( - rule="module_size_soft", - relative_path=rel_path, - identifier="module", - detail=f"lines={line_count}", - ) - ) - - return violations - - -def collect_alias_imports() -> list[Violation]: - """Collect alias import violations.""" - alias_pattern = re.compile(r"^import\s+(\w+)\s+as\s+(\w+)") - from_alias_pattern = re.compile(r"from\s+\S+\s+import\s+(\w+)\s+as\s+(\w+)") - - violations: list[Violation] = [] - - for py_file in find_source_files(): - content, error = read_file_safe(py_file) - if error or content is None: - continue - - rel_path = relative_path(py_file) - lines = content.splitlines() - - for i, line in enumerate(lines, start=1): - for pattern in [alias_pattern, from_alias_pattern]: - if match := pattern.search(line): - original, alias = match.groups() - if original.lower() not in alias.lower(): - if alias not in {"np", "pd", "plt", "tf", "nn", "F", "sa", "sd"}: - violations.append( - Violation( - rule="alias_import", - relative_path=rel_path, - identifier=content_hash(f"{i}:{line.strip()}"), - detail=f"{original}->{alias}", - ) - ) - - return violations - - -def collect_god_classes() -> list[Violation]: - """Collect god class violations.""" - max_methods = 15 - max_lines = 400 - violations: list[Violation] = [] - - for py_file in find_source_files(include_migrations=False): - tree, error = parse_file_safe(py_file) - if error or tree is None: - continue - - rel_path = relative_path(py_file) - - for node in ast.walk(tree): - if isinstance(node, ast.ClassDef): - methods = [ - n - for n in node.body - if isinstance(n, (ast.FunctionDef, ast.AsyncFunctionDef)) - ] - - if len(methods) > max_methods: - violations.append( - Violation( - rule="god_class", - relative_path=rel_path, - identifier=node.name, - detail=f"methods={len(methods)}", - ) - ) - - if node.end_lineno: - class_lines = node.end_lineno - node.lineno + 1 - if class_lines > max_lines: - violations.append( - Violation( - rule="god_class", - relative_path=rel_path, - identifier=node.name, - detail=f"lines={class_lines}", - ) - ) - - return violations - - -def collect_deep_nesting() -> list[Violation]: - """Collect deep nesting violations.""" - max_nesting = 2 - violations: list[Violation] = [] - - def count_nesting_depth(node: ast.AST, current_depth: int = 0) -> int: - max_depth = current_depth - nesting_nodes = ( - ast.If, ast.While, ast.For, ast.AsyncFor, - ast.With, ast.AsyncWith, ast.Try, - ast.FunctionDef, ast.AsyncFunctionDef, - ) - for child in ast.iter_child_nodes(node): - if isinstance(child, nesting_nodes): - child_depth = count_nesting_depth(child, current_depth + 1) - else: - child_depth = count_nesting_depth(child, current_depth) - max_depth = max(max_depth, child_depth) - return max_depth - - for py_file in find_source_files(include_migrations=False): - tree, error = parse_file_safe(py_file) - if error or tree is None: - continue - - rel_path = relative_path(py_file) - - for node in ast.walk(tree): - if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)): - depth = count_nesting_depth(node) - if depth > max_nesting: - violations.append( - Violation( - rule="deep_nesting", - relative_path=rel_path, - identifier=node.name, - detail=f"depth={depth}", - ) - ) - - return violations - - -def collect_feature_envy() -> list[Violation]: - """Collect feature envy violations.""" - excluded_class_patterns = { - "converter", - "exporter", - "repository", - } - excluded_method_patterns = { - "_to_domain", - "_to_proto", - "_proto_to_", - "_to_orm", - "_from_orm", - } - excluded_object_names = { - "model", - "meeting", - "segment", - "request", - "response", - "np", - "noteflow_pb2", - "seg", - "job", - "ai", - "summary", - "MeetingState", - "event", - "item", - "entity", - "uow", - "span", - "host", - } - - violations: list[Violation] = [] - - for py_file in find_source_files(include_migrations=False): - tree, error = parse_file_safe(py_file) - if error or tree is None: - continue - - rel_path = relative_path(py_file) - - for class_node in ast.walk(tree): - if not isinstance(class_node, ast.ClassDef): - continue - - if any(p in class_node.name.lower() for p in excluded_class_patterns): - continue - - for method in class_node.body: - if not isinstance(method, (ast.FunctionDef, ast.AsyncFunctionDef)): - continue - - if any(p in method.name.lower() for p in excluded_method_patterns): - continue - - self_accesses = 0 - other_accesses: dict[str, int] = {} - - for node in ast.walk(method): - if isinstance(node, ast.Attribute) and isinstance(node.value, ast.Name): - if node.value.id == "self": - self_accesses += 1 - else: - other_accesses[node.value.id] = other_accesses.get(node.value.id, 0) + 1 - - for other_obj, count in other_accesses.items(): - if other_obj in excluded_object_names: - continue - if count > self_accesses + 2 and count > 4: - violations.append( - Violation( - rule="feature_envy", - relative_path=rel_path, - identifier=f"{class_node.name}.{method.name}", - detail=f"{other_obj}={count}_vs_self={self_accesses}", - ) - ) - - return violations - - -def collect_redundant_type_aliases() -> list[Violation]: - """Collect redundant type alias violations.""" - violations: list[Violation] = [] - - for py_file in find_source_files(): - tree, error = parse_file_safe(py_file) - if error or tree is None: - continue - - rel_path = relative_path(py_file) - - for node in ast.walk(tree): - if isinstance(node, ast.AnnAssign) and isinstance(node.target, ast.Name): - if isinstance(node.annotation, ast.Name): - if node.annotation.id == "TypeAlias": - if isinstance(node.value, ast.Name): - base_type = node.value.id - if base_type in {"str", "int", "float", "bool", "bytes"}: - target_name = node.target.id - violations.append( - Violation( - rule="redundant_type_alias", - relative_path=rel_path, - identifier=target_name, - detail=base_type, - ) - ) - - return violations - - -def collect_passthrough_classes() -> list[Violation]: - """Collect passthrough class violations.""" - violations: list[Violation] = [] - - def is_thin_wrapper(node: ast.FunctionDef | ast.AsyncFunctionDef) -> str | None: - body_stmts = [ - s for s in node.body - if not isinstance(s, ast.Expr) or not isinstance(s.value, ast.Constant) - ] - if len(body_stmts) == 1: - stmt = body_stmts[0] - if isinstance(stmt, ast.Return) and isinstance(stmt.value, ast.Call): - call = stmt.value - if isinstance(call.func, ast.Name): - return call.func.id - elif isinstance(call.func, ast.Attribute): - return call.func.attr - return None - - for py_file in find_source_files(): - tree, error = parse_file_safe(py_file) - if error or tree is None: - continue - - rel_path = relative_path(py_file) - - for node in ast.walk(tree): - if isinstance(node, ast.ClassDef): - methods = [ - n for n in node.body - if isinstance(n, (ast.FunctionDef, ast.AsyncFunctionDef)) - and not n.name.startswith("_") - ] - - if len(methods) >= 3: - passthrough_count = sum( - 1 for m in methods if is_thin_wrapper(m) is not None - ) - - if passthrough_count == len(methods): - violations.append( - Violation( - rule="passthrough_class", - relative_path=rel_path, - identifier=node.name, - detail=f"{passthrough_count}_methods", - ) - ) - - return violations +from tests.quality._detectors.wrappers import ( + collect_alias_imports, + collect_passthrough_classes, + collect_redundant_type_aliases, + collect_thin_wrappers, +) def main() -> None: @@ -753,7 +110,9 @@ def main() -> None: save_baseline(all_violations) total = sum(len(v) for v in all_violations.values()) - print(f"\nBaseline generated with {total} total violations across {len(all_violations)} rules") + print( + f"\nBaseline generated with {total} total violations across {len(all_violations)} rules" + ) print(f"Saved to: {Path(__file__).parent / 'baselines.json'}") diff --git a/tests/quality/test_code_smells.py b/tests/quality/test_code_smells.py index b2a49e7..b99da89 100644 --- a/tests/quality/test_code_smells.py +++ b/tests/quality/test_code_smells.py @@ -11,97 +11,31 @@ Detects: from __future__ import annotations -import ast - from tests.quality._baseline import ( Violation, assert_no_new_violations, ) +from tests.quality._detectors.code_smells import ( + collect_deep_nesting, + collect_feature_envy, + collect_god_classes, + collect_high_complexity, + collect_long_methods, + collect_long_parameter_lists, + collect_module_size_soft, +) from tests.quality._helpers import ( collect_parse_errors, find_source_files, - parse_file_safe, read_file_safe, relative_path, ) -def count_branches(node: ast.AST) -> int: - """Count decision points (branches) in an AST node.""" - count = 0 - for child in ast.walk(node): - if isinstance( - child, (ast.If, ast.While, ast.For, ast.AsyncFor, ast.And, ast.Or) - ): - count += 1 - elif isinstance(child, ast.comprehension): - count += 1 - count += len(child.ifs) - elif isinstance(child, ast.ExceptHandler): - count += 1 - return count - - -def count_nesting_depth(node: ast.AST, current_depth: int = 0) -> int: - """Calculate maximum nesting depth.""" - max_depth = current_depth - - nesting_nodes = ( - ast.If, - ast.While, - ast.For, - ast.AsyncFor, - ast.With, - ast.AsyncWith, - ast.Try, - ast.FunctionDef, - ast.AsyncFunctionDef, - ) - - for child in ast.iter_child_nodes(node): - if isinstance(child, nesting_nodes): - child_depth = count_nesting_depth(child, current_depth + 1) - else: - child_depth = count_nesting_depth(child, current_depth) - max_depth = max(max_depth, child_depth) - - return max_depth - - -def count_function_lines(node: ast.FunctionDef | ast.AsyncFunctionDef) -> int: - """Count lines in a function body.""" - return 0 if node.end_lineno is None else node.end_lineno - node.lineno + 1 - - def test_no_high_complexity_functions() -> None: """Detect functions with high cyclomatic complexity.""" - max_complexity = 12 - - violations: list[Violation] = [] parse_errors: list[str] = [] - - for py_file in find_source_files(include_migrations=False): - tree, error = parse_file_safe(py_file) - if error: - parse_errors.append(error) - continue - if tree is None: - continue - - rel_path = relative_path(py_file) - - for node in ast.walk(tree): - if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)): - complexity = count_branches(node) + 1 - if complexity > max_complexity: - violations.append( - Violation( - rule="high_complexity", - relative_path=rel_path, - identifier=node.name, - detail=f"complexity={complexity}", - ) - ) + violations = collect_high_complexity(parse_errors=parse_errors) collect_parse_errors(parse_errors) assert_no_new_violations("high_complexity", violations) @@ -109,41 +43,8 @@ def test_no_high_complexity_functions() -> None: def test_no_long_parameter_lists() -> None: """Detect functions with too many parameters.""" - max_params = 4 - - violations: list[Violation] = [] parse_errors: list[str] = [] - - for py_file in find_source_files(include_migrations=False): - tree, error = parse_file_safe(py_file) - if error: - parse_errors.append(error) - continue - if tree is None: - continue - - rel_path = relative_path(py_file) - - for node in ast.walk(tree): - if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)): - args = node.args - total_params = ( - len(args.args) + len(args.posonlyargs) + len(args.kwonlyargs) - ) - if "self" in [a.arg for a in args.args]: - total_params -= 1 - if "cls" in [a.arg for a in args.args]: - total_params -= 1 - - if total_params > max_params: - violations.append( - Violation( - rule="long_parameter_list", - relative_path=rel_path, - identifier=node.name, - detail=f"params={total_params}", - ) - ) + violations = collect_long_parameter_lists(parse_errors=parse_errors) collect_parse_errors(parse_errors) assert_no_new_violations("long_parameter_list", violations) @@ -151,51 +52,8 @@ def test_no_long_parameter_lists() -> None: def test_no_god_classes() -> None: """Detect classes with too many methods or too much responsibility.""" - max_methods = 15 - max_lines = 400 - - violations: list[Violation] = [] parse_errors: list[str] = [] - - for py_file in find_source_files(include_migrations=False): - tree, error = parse_file_safe(py_file) - if error: - parse_errors.append(error) - continue - if tree is None: - continue - - rel_path = relative_path(py_file) - - for node in ast.walk(tree): - if isinstance(node, ast.ClassDef): - methods = [ - n - for n in node.body - if isinstance(n, (ast.FunctionDef, ast.AsyncFunctionDef)) - ] - - if len(methods) > max_methods: - violations.append( - Violation( - rule="god_class", - relative_path=rel_path, - identifier=node.name, - detail=f"methods={len(methods)}", - ) - ) - - if node.end_lineno: - class_lines = node.end_lineno - node.lineno + 1 - if class_lines > max_lines: - violations.append( - Violation( - rule="god_class", - relative_path=rel_path, - identifier=node.name, - detail=f"lines={class_lines}", - ) - ) + violations = collect_god_classes(parse_errors=parse_errors) collect_parse_errors(parse_errors) assert_no_new_violations("god_class", violations) @@ -203,33 +61,8 @@ def test_no_god_classes() -> None: def test_no_deep_nesting() -> None: """Detect functions with excessive nesting depth.""" - max_nesting = 2 - - violations: list[Violation] = [] parse_errors: list[str] = [] - - for py_file in find_source_files(include_migrations=False): - tree, error = parse_file_safe(py_file) - if error: - parse_errors.append(error) - continue - if tree is None: - continue - - rel_path = relative_path(py_file) - - for node in ast.walk(tree): - if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)): - depth = count_nesting_depth(node) - if depth > max_nesting: - violations.append( - Violation( - rule="deep_nesting", - relative_path=rel_path, - identifier=node.name, - detail=f"depth={depth}", - ) - ) + violations = collect_deep_nesting(parse_errors=parse_errors) collect_parse_errors(parse_errors) assert_no_new_violations("deep_nesting", violations) @@ -237,33 +70,8 @@ def test_no_deep_nesting() -> None: def test_no_long_methods() -> None: """Detect methods that are too long.""" - max_lines = 50 - - violations: list[Violation] = [] parse_errors: list[str] = [] - - for py_file in find_source_files(include_migrations=False): - tree, error = parse_file_safe(py_file) - if error: - parse_errors.append(error) - continue - if tree is None: - continue - - rel_path = relative_path(py_file) - - for node in ast.walk(tree): - if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)): - lines = count_function_lines(node) - if lines > max_lines: - violations.append( - Violation( - rule="long_method", - relative_path=rel_path, - identifier=node.name, - detail=f"lines={lines}", - ) - ) + violations = collect_long_methods(parse_errors=parse_errors) collect_parse_errors(parse_errors) assert_no_new_violations("long_method", violations) @@ -280,111 +88,8 @@ def test_no_feature_envy() -> None: These patterns are excluded from detection. """ - # Patterns that are NOT feature envy (they legitimately work with external objects) - excluded_class_patterns = { - "converter", - "exporter", - "repository", - } - excluded_method_patterns = { - "_to_domain", - "_to_proto", - "_proto_to_", - "_to_orm", - "_from_orm", - } - # Objects that are commonly used more than self but aren't feature envy - excluded_object_names = { - "model", # ORM model in repo methods - "meeting", # Domain entity in exporters - "segment", # Domain entity in converters - "request", # gRPC request in handlers - "response", # gRPC response in handlers - "np", # numpy operations - "noteflow_pb2", # protobuf module - "seg", # Loop iteration over segments - "job", # Job processing in background tasks - "repo", # Repository access in service methods - "ai", # AI response processing - "summary", # Summary processing in verification - "MeetingState", # Enum class methods - "event", # Usage event, calendar event processing - "item", # API response item parsing - "entity", # Named entity processing - "uow", # Unit of work in service methods - "span", # OpenTelemetry span in observability - "host", # Servicer host in mixin methods - } - - def _is_excluded_class(class_name: str) -> bool: - """Check if class should be excluded from feature envy detection.""" - class_name_lower = class_name.lower() - return any(p in class_name_lower for p in excluded_class_patterns) - - def _is_excluded_method(method_name: str) -> bool: - """Check if method should be excluded from feature envy detection.""" - method_name_lower = method_name.lower() - return any(p in method_name_lower for p in excluded_method_patterns) - - def _count_accesses( - method: ast.FunctionDef | ast.AsyncFunctionDef, - ) -> tuple[int, dict[str, int]]: - """Count self accesses and other object accesses in a method.""" - self_accesses = 0 - other_accesses: dict[str, int] = {} - - for node in ast.walk(method): - if isinstance(node, ast.Attribute) and isinstance(node.value, ast.Name): - if node.value.id == "self": - self_accesses += 1 - else: - other_accesses[node.value.id] = ( - other_accesses.get(node.value.id, 0) + 1 - ) - - return self_accesses, other_accesses - - violations: list[Violation] = [] parse_errors: list[str] = [] - - for py_file in find_source_files(include_migrations=False): - tree, error = parse_file_safe(py_file) - if error: - parse_errors.append(error) - continue - if tree is None: - continue - - rel_path = relative_path(py_file) - - for class_node in ast.walk(tree): - if not isinstance(class_node, ast.ClassDef): - continue - - if _is_excluded_class(class_node.name): - continue - - for method in class_node.body: - if not isinstance(method, (ast.FunctionDef, ast.AsyncFunctionDef)): - continue - - if _is_excluded_method(method.name): - continue - - self_accesses, other_accesses = _count_accesses(method) - - for other_obj, count in other_accesses.items(): - if other_obj in excluded_object_names: - continue - if count > self_accesses + 2 and count > 4: - violations.append( - Violation( - rule="feature_envy", - relative_path=rel_path, - identifier=f"{class_node.name}.{method.name}", - detail=f"{other_obj}={count}_vs_self={self_accesses}", - ) - ) + violations = collect_feature_envy(parse_errors=parse_errors) collect_parse_errors(parse_errors) assert_no_new_violations("feature_envy", violations) @@ -395,7 +100,6 @@ def test_module_size_limits() -> None: soft_limit = 350 hard_limit = 600 - soft_violations: list[Violation] = [] hard_violations: list[Violation] = [] for py_file in find_source_files(include_migrations=False): @@ -415,15 +119,8 @@ def test_module_size_limits() -> None: detail=f"lines={line_count}", ) ) - elif line_count > soft_limit: - soft_violations.append( - Violation( - rule="module_size_soft", - relative_path=rel_path, - identifier="module", - detail=f"lines={line_count}", - ) - ) + + soft_violations = collect_module_size_soft(soft_limit=soft_limit) # Hard limit: zero tolerance assert not hard_violations, ( diff --git a/tests/quality/test_stale_code.py b/tests/quality/test_stale_code.py index 326dd7d..d359bc5 100644 --- a/tests/quality/test_stale_code.py +++ b/tests/quality/test_stale_code.py @@ -16,7 +16,11 @@ import re from tests.quality._baseline import ( Violation, assert_no_new_violations, - content_hash, +) +from tests.quality._detectors.stale_code import ( + collect_deprecated_patterns, + collect_orphaned_imports, + collect_stale_todos, ) from tests.quality._helpers import ( collect_parse_errors, @@ -29,33 +33,7 @@ from tests.quality._helpers import ( def test_no_stale_todos() -> None: """Detect old TODO/FIXME comments that should be addressed.""" - stale_pattern = re.compile( - r"#\s*(TODO|FIXME|HACK|XXX|DEPRECATED)[\s:]*(.{0,100})", - re.IGNORECASE, - ) - - violations: list[Violation] = [] - - for py_file in find_source_files(): - content, error = read_file_safe(py_file) - if error or content is None: - continue - - rel_path = relative_path(py_file) - lines = content.splitlines() - - for i, line in enumerate(lines, start=1): - if match := stale_pattern.search(line): - tag = match[1].upper() - message = match[2].strip()[:50] - violations.append( - Violation( - rule="stale_todo", - relative_path=rel_path, - identifier=content_hash(f"{i}:{line.strip()}"), - detail=f"{tag}:{message}", - ) - ) + violations = collect_stale_todos() assert_no_new_violations("stale_todo", violations) @@ -119,74 +97,8 @@ def test_no_orphaned_imports() -> None: Note: Skips __init__.py files since re-exports are intentional public API. """ - violations: list[Violation] = [] parse_errors: list[str] = [] - - for py_file in find_source_files(include_init=False): - tree, error = parse_file_safe(py_file) - if error: - parse_errors.append(error) - continue - if tree is None: - continue - - rel_path = relative_path(py_file) - - imported_names: set[str] = set() - for node in ast.walk(tree): - if isinstance(node, ast.Import): - for alias in node.names: - name = alias.asname or alias.name.split(".")[0] - imported_names.add(name) - elif isinstance(node, ast.ImportFrom): - for alias in node.names: - if alias.name != "*": - name = alias.asname or alias.name - imported_names.add(name) - - used_names: set[str] = set() - for node in ast.walk(tree): - if isinstance(node, ast.Name): - used_names.add(node.id) - elif isinstance(node, ast.Attribute) and isinstance(node.value, ast.Name): - used_names.add(node.value.id) - - # Check for __all__ re-exports (names listed in __all__ are considered used) - all_exports: set[str] = set() - for node in ast.walk(tree): - if isinstance(node, ast.Assign): - for target in node.targets: - if isinstance(target, ast.Name) and target.id == "__all__": - if isinstance(node.value, ast.List): - for elt in node.value.elts: - if isinstance(elt, ast.Constant) and isinstance( - elt.value, str - ): - all_exports.add(elt.value) - - # Check for TYPE_CHECKING imports - type_checking_imports: set[str] = set() - for node in ast.walk(tree): - if isinstance(node, ast.If): - if isinstance(node.test, ast.Name) and node.test.id == "TYPE_CHECKING": - for subnode in ast.walk(node): - if isinstance(subnode, ast.ImportFrom): - for alias in subnode.names: - name = alias.asname or alias.name - type_checking_imports.add(name) - - unused = imported_names - used_names - type_checking_imports - all_exports - unused -= {"__future__", "annotations"} - - violations.extend( - Violation( - rule="orphaned_import", - relative_path=rel_path, - identifier=name, - ) - for name in sorted(unused) - if not name.startswith("_") - ) + violations = collect_orphaned_imports(parse_errors=parse_errors) collect_parse_errors(parse_errors) assert_no_new_violations("orphaned_import", violations) @@ -232,36 +144,5 @@ def test_no_unreachable_code() -> None: def test_no_deprecated_patterns() -> None: """Detect usage of deprecated patterns and APIs.""" - deprecated_patterns = [ - (r"\.format\s*\(", "str.format()"), - (r"% \(", "%-formatting"), - (r"from typing import Optional", "Optional[X]"), - (r"from typing import Union", "Union[X, Y]"), - (r"from typing import List\b", "List[X]"), - (r"from typing import Dict\b", "Dict[K, V]"), - (r"from typing import Tuple\b", "Tuple[X, ...]"), - (r"from typing import Set\b", "Set[X]"), - ] - - violations: list[Violation] = [] - - for py_file in find_source_files(): - content, error = read_file_safe(py_file) - if error or content is None: - continue - - rel_path = relative_path(py_file) - lines = content.splitlines() - - for i, line in enumerate(lines, start=1): - violations.extend( - Violation( - rule="deprecated_pattern", - relative_path=rel_path, - identifier=content_hash(f"{i}:{line.strip()}"), - detail=old_style, - ) - for pattern, old_style in deprecated_patterns - if re.search(pattern, line) - ) + violations = collect_deprecated_patterns() assert_no_new_violations("deprecated_pattern", violations) diff --git a/tests/quality/test_test_smells.py b/tests/quality/test_test_smells.py index a9a56ae..f73db9f 100644 --- a/tests/quality/test_test_smells.py +++ b/tests/quality/test_test_smells.py @@ -29,6 +29,27 @@ from tests.quality._baseline import ( assert_no_new_violations, content_hash, ) +from tests.quality._detectors.test_smells import ( + collect_assertion_roulette, + collect_conditional_test_logic, + collect_duplicate_test_names, + collect_eager_tests, + collect_exception_handling, + collect_fixture_missing_type, + collect_fixture_scope_too_narrow, + collect_long_tests, + collect_magic_number_tests, + collect_raises_without_match, + collect_redundant_prints, + collect_sensitive_equality, + collect_sleepy_tests, + collect_unknown_tests, + collect_unused_fixtures, + get_fixture_scope, + get_fixtures, + get_module_level_fixtures, + get_test_methods, +) from tests.quality._helpers import ( collect_parse_errors, find_test_files, @@ -38,94 +59,19 @@ from tests.quality._helpers import ( ) -def get_test_methods(tree: ast.AST) -> list[ast.FunctionDef | ast.AsyncFunctionDef]: - """Extract test methods from AST.""" - tests: list[ast.FunctionDef | ast.AsyncFunctionDef] = [] - for node in ast.walk(tree): - if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)): - if node.name.startswith("test_"): - tests.append(node) - return tests - - -def count_assertions(node: ast.AST) -> int: - """Count assertion statements in a node.""" - count = 0 - for child in ast.walk(node): - if isinstance(child, ast.Assert): - count += 1 - elif isinstance(child, ast.Call): - if isinstance(child.func, ast.Attribute): - if child.func.attr in { - "assertEqual", - "assertNotEqual", - "assertTrue", - "assertFalse", - "assertIs", - "assertIsNot", - "assertIsNone", - "assertIsNotNone", - "assertIn", - "assertNotIn", - "assertRaises", - "assertWarns", - }: - count += 1 - return count - - -def has_assertion_message(node: ast.Assert) -> bool: - """Check if an assert statement has a message.""" - return node.msg is not None - - def test_no_assertion_roulette() -> None: """Detect tests with multiple assertions without descriptive messages. Assertion Roulette occurs when a test has multiple assertions without messages, making it hard to determine which assertion failed. """ - violations: list[Violation] = [] parse_errors: list[str] = [] - - for py_file in find_test_files(): - tree, error = parse_file_safe(py_file) - if error: - parse_errors.append(error) - continue - if tree is None: - continue - - rel_path = relative_path(py_file) - - for test_method in get_test_methods(tree): - assertions_without_msg = 0 - - for node in ast.walk(test_method): - if isinstance(node, ast.Assert): - if not has_assertion_message(node): - assertions_without_msg += 1 - - # Flag if >1 assertions without messages - if assertions_without_msg > 1: - violations.append( - Violation( - rule="assertion_roulette", - relative_path=rel_path, - identifier=test_method.name, - detail=f"assertions={assertions_without_msg}", - ) - ) + violations = collect_assertion_roulette(parse_errors=parse_errors) collect_parse_errors(parse_errors) assert_no_new_violations("assertion_roulette", violations) -def _contains_assertion(node: ast.AST) -> bool: - """Check if a node contains an assertion statement.""" - return any(isinstance(child, ast.Assert) for child in ast.walk(node)) - - def test_no_conditional_test_logic() -> None: """Detect tests containing conditional logic (if/for/while) with assertions inside. @@ -135,44 +81,8 @@ def test_no_conditional_test_logic() -> None: Note: Loops/conditionals used only for setup (without assertions) are allowed. Stress tests are excluded as they intentionally use loops for thorough testing. """ - violations: list[Violation] = [] parse_errors: list[str] = [] - - for py_file in find_test_files(): - # Skip stress tests - they intentionally use loops for thorough testing - if "stress" in py_file.parts: - continue - - tree, error = parse_file_safe(py_file) - if error: - parse_errors.append(error) - continue - if tree is None: - continue - - rel_path = relative_path(py_file) - - for test_method in get_test_methods(tree): - conditionals: list[str] = [] - - for node in ast.walk(test_method): - # Only flag conditionals that contain assertions inside them - if isinstance(node, ast.If) and _contains_assertion(node): - conditionals.append(f"if@{node.lineno}") - elif isinstance(node, ast.For) and _contains_assertion(node): - conditionals.append(f"for@{node.lineno}") - elif isinstance(node, ast.While) and _contains_assertion(node): - conditionals.append(f"while@{node.lineno}") - - if conditionals: - violations.append( - Violation( - rule="conditional_test_logic", - relative_path=rel_path, - identifier=test_method.name, - detail=",".join(conditionals[:3]), - ) - ) + violations = collect_conditional_test_logic(parse_errors=parse_errors) collect_parse_errors(parse_errors) assert_no_new_violations("conditional_test_logic", violations) @@ -234,43 +144,8 @@ def test_no_sleepy_tests() -> None: - tests/integration/test_database_resilience.py: Tests connection pool timing - tests/grpc/test_stream_lifecycle.py: Tests streaming cleanup timing """ - # Paths where sleep is legitimately needed for stress/resilience testing - allowed_sleepy_paths = { - "tests/stress/", - } - - violations: list[Violation] = [] parse_errors: list[str] = [] - - for py_file in find_test_files(): - # Skip files where sleep is legitimately needed for stress/resilience testing - rel_path = relative_path(py_file) - if any(allowed in rel_path for allowed in allowed_sleepy_paths): - continue - - tree, error = parse_file_safe(py_file) - if error: - parse_errors.append(error) - continue - if tree is None: - continue - - for test_method in get_test_methods(tree): - for node in ast.walk(test_method): - if isinstance(node, ast.Call): - # Check for time.sleep() - if isinstance(node.func, ast.Attribute): - if node.func.attr == "sleep": - if isinstance(node.func.value, ast.Name): - if node.func.value.id in {"time", "asyncio"}: - violations.append( - Violation( - rule="sleepy_test", - relative_path=rel_path, - identifier=test_method.name, - detail=f"line={node.lineno}", - ) - ) + violations = collect_sleepy_tests(parse_errors=parse_errors) collect_parse_errors(parse_errors) assert_no_new_violations("sleepy_test", violations) @@ -282,48 +157,8 @@ def test_no_unknown_tests() -> None: Tests without assertions pass even when behavior is wrong, providing false confidence. Every test should assert something. """ - violations: list[Violation] = [] parse_errors: list[str] = [] - - for py_file in find_test_files(): - tree, error = parse_file_safe(py_file) - if error: - parse_errors.append(error) - continue - if tree is None: - continue - - rel_path = relative_path(py_file) - - for test_method in get_test_methods(tree): - has_assertion = False - has_raises = False - - for node in ast.walk(test_method): - if isinstance(node, ast.Assert): - has_assertion = True - break - # Check for pytest.raises context manager - if isinstance(node, ast.With): - for item in node.items: - if isinstance(item.context_expr, ast.Call): - call = item.context_expr - if isinstance(call.func, ast.Attribute): - if call.func.attr in {"raises", "warns"}: - has_raises = True - - if not has_assertion and not has_raises: - # Check if it's a smoke test (just calling a function) - # These are valid for checking no exceptions are raised - has_call = any(isinstance(n, ast.Call) for n in ast.walk(test_method)) - if not has_call: - violations.append( - Violation( - rule="unknown_test", - relative_path=rel_path, - identifier=test_method.name, - ) - ) + violations = collect_unknown_tests(parse_errors=parse_errors) collect_parse_errors(parse_errors) assert_no_new_violations("unknown_test", violations) @@ -394,31 +229,8 @@ def test_no_redundant_prints() -> None: Print statements in tests are noise during automated runs. Use logging or pytest's capture mechanism instead. """ - violations: list[Violation] = [] parse_errors: list[str] = [] - - for py_file in find_test_files(): - tree, error = parse_file_safe(py_file) - if error: - parse_errors.append(error) - continue - if tree is None: - continue - - rel_path = relative_path(py_file) - - for test_method in get_test_methods(tree): - for node in ast.walk(test_method): - if isinstance(node, ast.Call) and isinstance(node.func, ast.Name): - if node.func.id == "print": - violations.append( - Violation( - rule="redundant_print", - relative_path=rel_path, - identifier=test_method.name, - detail=f"line={node.lineno}", - ) - ) + violations = collect_redundant_prints(parse_errors=parse_errors) collect_parse_errors(parse_errors) assert_no_new_violations("redundant_print", violations) @@ -484,43 +296,8 @@ def test_no_exception_handling_in_tests() -> None: Tests should use pytest.raises() for expected exceptions, not try/except. Manual exception handling can hide bugs. """ - violations: list[Violation] = [] parse_errors: list[str] = [] - - for py_file in find_test_files(): - tree, error = parse_file_safe(py_file) - if error: - parse_errors.append(error) - continue - if tree is None: - continue - - rel_path = relative_path(py_file) - - for test_method in get_test_methods(tree): - for node in ast.walk(test_method): - if isinstance(node, ast.Try): - # Check if it's a bare except or catching specific exceptions - for handler in node.handlers: - if handler.type is None: - violations.append( - Violation( - rule="exception_handling", - relative_path=rel_path, - identifier=test_method.name, - detail="bare_except", - ) - ) - elif isinstance(handler.type, ast.Name): - if handler.type.id in {"Exception", "BaseException"}: - violations.append( - Violation( - rule="exception_handling", - relative_path=rel_path, - identifier=test_method.name, - detail=f"catches_{handler.type.id}", - ) - ) + violations = collect_exception_handling(parse_errors=parse_errors) collect_parse_errors(parse_errors) assert_no_new_violations("exception_handling", violations) @@ -532,39 +309,8 @@ def test_no_magic_numbers_in_assertions() -> None: Assertions like `assert result == 42` are unclear. Use named constants or variables with descriptive names. """ - # Allowed magic numbers in tests - allowed_numbers = {0, 1, 2, -1, 100, 1000} - - violations: list[Violation] = [] parse_errors: list[str] = [] - - for py_file in find_test_files(): - tree, error = parse_file_safe(py_file) - if error: - parse_errors.append(error) - continue - if tree is None: - continue - - rel_path = relative_path(py_file) - - for test_method in get_test_methods(tree): - for node in ast.walk(test_method): - if isinstance(node, ast.Assert): - # Find numeric literals in assertions - for child in ast.walk(node): - if isinstance(child, ast.Constant): - if isinstance(child.value, (int, float)): - if child.value not in allowed_numbers: - if abs(float(child.value)) > 10.0: - violations.append( - Violation( - rule="magic_number_test", - relative_path=rel_path, - identifier=test_method.name, - detail=f"value={child.value}", - ) - ) + violations = collect_magic_number_tests(parse_errors=parse_errors) collect_parse_errors(parse_errors) assert_no_new_violations("magic_number_test", violations) @@ -582,65 +328,8 @@ def test_no_sensitive_equality() -> None: - Tests with "conversion" in name - testing type conversion - Tests with "serializ" in name - testing serialization """ - # Test names that legitimately use str() comparisons - excluded_test_patterns = { - "string", # Testing string conversion behavior - "proto", # Testing protobuf field serialization - } - - # File patterns where str() comparison is expected (gRPC field serialization) - excluded_file_patterns: set[str] = set() - - def _is_excluded_test(test_name: str) -> bool: - """Check if test legitimately uses str() comparison.""" - test_name_lower = test_name.lower() - return any(p in test_name_lower for p in excluded_test_patterns) - - def _is_excluded_file(file_path: Path) -> bool: - """Check if file legitimately uses str() comparisons.""" - file_name_lower = file_path.stem.lower() - return any(p in file_name_lower for p in excluded_file_patterns) - - violations: list[Violation] = [] parse_errors: list[str] = [] - - for py_file in find_test_files(): - # Skip files that legitimately use str() comparisons - if _is_excluded_file(py_file): - continue - - tree, error = parse_file_safe(py_file) - if error: - parse_errors.append(error) - continue - if tree is None: - continue - - rel_path = relative_path(py_file) - - for test_method in get_test_methods(tree): - # Skip tests that legitimately test string behavior - if _is_excluded_test(test_method.name): - continue - for node in ast.walk(test_method): - if isinstance(node, ast.Assert): - # Check for str(x) == "..." or repr(x) == "..." as direct comparators - # Only flag when str()/repr() is a direct operand, not nested in other calls - if isinstance(node.test, ast.Compare): - # Check left operand and all comparators (right operands) - operands = [node.test.left, *node.test.comparators] - for operand in operands: - if isinstance(operand, ast.Call): - if isinstance(operand.func, ast.Name): - if operand.func.id in {"str", "repr"}: - violations.append( - Violation( - rule="sensitive_equality", - relative_path=rel_path, - identifier=test_method.name, - detail=operand.func.id, - ) - ) + violations = collect_sensitive_equality(parse_errors=parse_errors) collect_parse_errors(parse_errors) assert_no_new_violations("sensitive_equality", violations) @@ -652,39 +341,8 @@ def test_no_eager_tests() -> None: Eager tests are hard to maintain because failures don't pinpoint the problem. Each test should focus on one behavior. """ - violations: list[Violation] = [] - parse_errors: list[str] = [] - max_method_calls = 7 # Threshold for "too many" method calls - - for py_file in find_test_files(): - tree, error = parse_file_safe(py_file) - if error: - parse_errors.append(error) - continue - if tree is None: - continue - - rel_path = relative_path(py_file) - - for test_method in get_test_methods(tree): - method_calls: set[str] = set() - - for node in ast.walk(test_method): - if isinstance(node, ast.Call): - if isinstance(node.func, ast.Attribute): - # Skip assert methods and common test utilities - if not node.func.attr.startswith("assert"): - method_calls.add(node.func.attr) - - if len(method_calls) > max_method_calls: - violations.append( - Violation( - rule="eager_test", - relative_path=rel_path, - identifier=test_method.name, - detail=f"methods={len(method_calls)}", - ) - ) + parse_errors: list[str] = [] + violations = collect_eager_tests(parse_errors=parse_errors) collect_parse_errors(parse_errors) assert_no_new_violations("eager_test", violations) @@ -696,34 +354,8 @@ def test_no_duplicate_test_names() -> None: Duplicate test names can cause confusion and may result in tests being shadowed or not run. """ - test_names: dict[str, list[tuple[Path, int]]] = defaultdict(list) parse_errors: list[str] = [] - - for py_file in find_test_files(): - tree, error = parse_file_safe(py_file) - if error: - parse_errors.append(error) - continue - if tree is None: - continue - - for test_method in get_test_methods(tree): - test_names[test_method.name].append((py_file, test_method.lineno)) - - duplicates = {name: locs for name, locs in test_names.items() if len(locs) > 1} - - violations: list[Violation] = [] - for name, locs in duplicates.items(): - # Use first location as the identifier - first_path = relative_path(locs[0][0]) - violations.append( - Violation( - rule="duplicate_test_name", - relative_path=first_path, - identifier=name, - detail=f"count={len(locs)}", - ) - ) + violations = collect_duplicate_test_names(parse_errors=parse_errors) collect_parse_errors(parse_errors) assert_no_new_violations("duplicate_test_name", violations) @@ -787,33 +419,8 @@ def test_no_long_test_methods() -> None: Long tests are hard to understand and maintain. Break them into smaller, focused tests or extract helper functions. """ - max_lines = 35 - - violations: list[Violation] = [] parse_errors: list[str] = [] - - for py_file in find_test_files(): - tree, error = parse_file_safe(py_file) - if error: - parse_errors.append(error) - continue - if tree is None: - continue - - rel_path = relative_path(py_file) - - for test_method in get_test_methods(tree): - if test_method.end_lineno: - lines = test_method.end_lineno - test_method.lineno + 1 - if lines > max_lines: - violations.append( - Violation( - rule="long_test", - relative_path=rel_path, - identifier=test_method.name, - detail=f"lines={lines}", - ) - ) + violations = collect_long_tests(parse_errors=parse_errors) collect_parse_errors(parse_errors) assert_no_new_violations("long_test", violations) @@ -824,64 +431,6 @@ def test_no_long_test_methods() -> None: # ============================================================================= -def _get_fixtures(tree: ast.AST) -> list[ast.FunctionDef]: - """Extract pytest fixtures from AST.""" - fixtures: list[ast.FunctionDef] = [] - for node in ast.walk(tree): - if isinstance(node, ast.FunctionDef): - for decorator in node.decorator_list: - # Check for @pytest.fixture or @fixture - if isinstance(decorator, ast.Attribute): - if decorator.attr == "fixture": - fixtures.append(node) - break - elif isinstance(decorator, ast.Call): - if isinstance(decorator.func, ast.Attribute): - if decorator.func.attr == "fixture": - fixtures.append(node) - break - elif isinstance(decorator, ast.Name) and decorator.id == "fixture": - fixtures.append(node) - break - return fixtures - - -def _get_fixture_scope(node: ast.FunctionDef) -> str | None: - """Extract fixture scope from decorator.""" - for decorator in node.decorator_list: - if isinstance(decorator, ast.Call): - for keyword in decorator.keywords: - if keyword.arg == "scope": - if isinstance(keyword.value, ast.Constant): - return str(keyword.value.value) - return None - - -def _get_module_level_fixtures(tree: ast.AST) -> list[ast.FunctionDef]: - """Extract only module-level pytest fixtures from AST (not class-scoped).""" - fixtures: list[ast.FunctionDef] = [] - # Only check top-level function definitions, not methods inside classes - if not isinstance(tree, ast.Module): - return fixtures - for node in tree.body: - if isinstance(node, ast.FunctionDef): - for decorator in node.decorator_list: - # Check for @pytest.fixture or @fixture - if isinstance(decorator, ast.Attribute): - if decorator.attr == "fixture": - fixtures.append(node) - break - elif isinstance(decorator, ast.Call): - if isinstance(decorator.func, ast.Attribute): - if decorator.func.attr == "fixture": - fixtures.append(node) - break - elif isinstance(decorator, ast.Name) and decorator.id == "fixture": - fixtures.append(node) - break - return fixtures - - def test_no_unittest_style_assertions() -> None: """Detect unittest-style assertions instead of plain assert. @@ -989,8 +538,8 @@ def test_no_session_scoped_fixtures_with_mutation() -> None: rel_path = relative_path(py_file) - for fixture in _get_fixtures(tree): - scope = _get_fixture_scope(fixture) + for fixture in get_fixtures(tree): + scope = get_fixture_scope(fixture) if scope in ("session", "module"): if fixture_source := ast.get_source_segment(content, fixture): for pattern in mutation_patterns: @@ -1020,31 +569,8 @@ def test_fixtures_have_type_hints() -> None: Fixtures should have return type annotations for better IDE support and documentation. Use -> T or -> Generator[T, None, None] for yields. """ - violations: list[Violation] = [] parse_errors: list[str] = [] - - for py_file in find_test_files(): - tree, error = parse_file_safe(py_file) - if error: - parse_errors.append(error) - continue - if tree is None: - continue - - rel_path = relative_path(py_file) - - for fixture in _get_fixtures(tree): - # Check if fixture has return type annotation - if fixture.returns is None: - # Skip fixtures that start with _ (internal helpers) - if not fixture.name.startswith("_"): - violations.append( - Violation( - rule="fixture_missing_type", - relative_path=rel_path, - identifier=fixture.name, - ) - ) + violations = collect_fixture_missing_type(parse_errors=parse_errors) collect_parse_errors(parse_errors) assert_no_new_violations("fixture_missing_type", violations) @@ -1056,62 +582,8 @@ def test_no_unused_fixture_parameters() -> None: Requesting unused fixtures wastes resources and clutters the test signature. Remove unused fixture parameters or mark them with underscore prefix. """ - violations: list[Violation] = [] parse_errors: list[str] = [] - - for py_file in find_test_files(): - tree, error = parse_file_safe(py_file) - if error: - parse_errors.append(error) - continue - if tree is None: - continue - - rel_path = relative_path(py_file) - - for test_method in get_test_methods(tree): - # Get parameter names (excluding self) - params = [ - arg.arg - for arg in test_method.args.args - if arg.arg not in ("self", "cls") - ] - - # Skip parameters that start with _ (explicitly unused) - params = [p for p in params if not p.startswith("_")] - - used_names: set[str] = { - node.id - for node in ast.walk(test_method) - if isinstance(node, ast.Name) - } - # Find unused parameters - for param in params: - if param not in used_names: - # Skip common pytest fixtures that have side effects - if param in ( - "monkeypatch", - "capsys", - "capfd", - "caplog", - "tmp_path", - "tmp_path_factory", - "request", - "pytestconfig", - "record_property", - "record_testsuite_property", - "recwarn", - "event_loop", - ): - continue - violations.append( - Violation( - rule="unused_fixture", - relative_path=rel_path, - identifier=test_method.name, - detail=param, - ) - ) + violations = collect_unused_fixtures(parse_errors=parse_errors) collect_parse_errors(parse_errors) assert_no_new_violations("unused_fixture", violations) @@ -1144,7 +616,7 @@ def test_conftest_fixtures_not_duplicated() -> None: continue conftest_dir = conftest.parent conftest_by_dir[conftest_dir] = { - fixture.name: fixture.lineno for fixture in _get_fixtures(tree) + fixture.name: fixture.lineno for fixture in get_fixtures(tree) } # Check test files for duplicate fixture definitions violations: list[Violation] = [] @@ -1175,7 +647,7 @@ def test_conftest_fixtures_not_duplicated() -> None: continue # Only check module-level fixtures (class-scoped fixtures are intentional) - for fixture in _get_module_level_fixtures(tree): + for fixture in get_module_level_fixtures(tree): if fixture.name in visible_conftest_fixtures: violations.append( Violation( @@ -1201,50 +673,8 @@ def test_fixture_scope_appropriate() -> None: - Function-scoped fixtures that create expensive resources should use module/session - Session-scoped fixtures that yield mutable objects should use function scope """ - # Patterns suggesting expensive setup - expensive_patterns = [ - r"asyncpg\.connect", - r"create_async_engine", - r"aiohttp\.ClientSession", - r"httpx\.AsyncClient", - r"subprocess\.Popen", - r"docker\.", - r"testcontainers\.", - ] - - violations: list[Violation] = [] parse_errors: list[str] = [] - - for py_file in find_test_files(): - tree, error = parse_file_safe(py_file) - if error: - parse_errors.append(error) - continue - if tree is None: - continue - - content, _ = read_file_safe(py_file) - if content is None: - continue - - rel_path = relative_path(py_file) - - for fixture in _get_fixtures(tree): - scope = _get_fixture_scope(fixture) - if fixture_source := ast.get_source_segment(content, fixture): - # Check for expensive operations in function-scoped fixtures - if scope is None or scope == "function": - for pattern in expensive_patterns: - if re.search(pattern, fixture_source): - violations.append( - Violation( - rule="fixture_scope_too_narrow", - relative_path=rel_path, - identifier=fixture.name, - detail="expensive_setup", - ) - ) - break + violations = collect_fixture_scope_too_narrow(parse_errors=parse_errors) collect_parse_errors(parse_errors) assert_no_new_violations("fixture_scope_too_narrow", violations) @@ -1256,40 +686,8 @@ def test_no_pytest_raises_without_match() -> None: Using pytest.raises without match= can hide bugs where the wrong exception is raised. Always specify match= to verify the exception message. """ - violations: list[Violation] = [] parse_errors: list[str] = [] - - for py_file in find_test_files(): - tree, error = parse_file_safe(py_file) - if error: - parse_errors.append(error) - continue - if tree is None: - continue - - rel_path = relative_path(py_file) - - for test_method in get_test_methods(tree): - for node in ast.walk(test_method): - if isinstance(node, ast.Call): - # Check for pytest.raises - if isinstance(node.func, ast.Attribute): - if node.func.attr == "raises": - if isinstance(node.func.value, ast.Name): - if node.func.value.id == "pytest": - # Check if match= is provided - has_match = any( - kw.arg == "match" for kw in node.keywords - ) - if not has_match: - violations.append( - Violation( - rule="raises_without_match", - relative_path=rel_path, - identifier=test_method.name, - detail=f"line={node.lineno}", - ) - ) + violations = collect_raises_without_match(parse_errors=parse_errors) collect_parse_errors(parse_errors) assert_no_new_violations("raises_without_match", violations) @@ -1315,7 +713,7 @@ def test_no_cross_file_fixture_duplicates() -> None: if tree is None: continue - for fixture in _get_module_level_fixtures(tree): + for fixture in get_module_level_fixtures(tree): fixture_locations[fixture.name].append((py_file, fixture.lineno)) # Find fixtures defined in multiple files diff --git a/tests/quality/test_unnecessary_wrappers.py b/tests/quality/test_unnecessary_wrappers.py index d52cfc2..60931a7 100644 --- a/tests/quality/test_unnecessary_wrappers.py +++ b/tests/quality/test_unnecessary_wrappers.py @@ -9,42 +9,20 @@ Detects: from __future__ import annotations -import ast -import re - from tests.quality._baseline import ( - Violation, assert_no_new_violations, - content_hash, +) +from tests.quality._detectors.wrappers import ( + collect_alias_imports, + collect_passthrough_classes, + collect_redundant_type_aliases, + collect_thin_wrappers, ) from tests.quality._helpers import ( collect_parse_errors, - find_source_files, - parse_file_safe, - read_file_safe, - relative_path, ) -def is_thin_wrapper(node: ast.FunctionDef | ast.AsyncFunctionDef) -> str | None: - """Check if function is a thin wrapper returning another call directly.""" - body_stmts = [ - s - for s in node.body - if not isinstance(s, ast.Expr) or not isinstance(s.value, ast.Constant) - ] - - if len(body_stmts) == 1: - stmt = body_stmts[0] - if isinstance(stmt, ast.Return) and isinstance(stmt.value, ast.Call): - call = stmt.value - if isinstance(call.func, ast.Name): - return call.func.id - elif isinstance(call.func, ast.Attribute): - return call.func.attr - return None - - def test_no_trivial_wrapper_functions() -> None: """Detect functions that simply wrap another function call. @@ -54,86 +32,8 @@ def test_no_trivial_wrapper_functions() -> None: - Factory methods that use cls() pattern - Domain methods that provide semantic meaning (is_active -> property check) """ - # Valid wrapper patterns that should be allowed - allowed_wrappers = { - # Public API facades - ("get_settings", "_load_settings"), - ("get_trigger_settings", "_load_trigger_settings"), - # Cached settings loaders (@lru_cache provides memoization) - ("get_feature_flags", "model_validate"), - ("get_calendar_settings", "model_validate"), - # Factory patterns - ("from_args", "cls"), - # Properties that add semantic meaning - ("segment_count", "len"), - ("full_transcript", "join"), - ("duration", "sub"), - ("is_active", "property"), - # Domain method accessors (type-safe dict access) - ("get_metadata", "get"), - # Strategy pattern implementations (RuleType.evaluate for simple mode) - ("evaluate", "RuleResult"), - # Type conversions - ("database_url_str", "str"), - ("generate_request_id", "str"), # UUID to string conversion - # Context variable accessors (public API over internal contextvars) - ("get_request_id", "get"), - # Time conversion utilities (semantic naming for datetime operations) - ("datetime_to_epoch_seconds", "timestamp"), - ("datetime_to_iso_string", "isoformat"), - # Accessor-style wrappers with semantic names - ("from_metrics", "cls"), - ("from_dict", "cls"), - # ProcessingStepState factory methods (GAP-W05) - ("pending", "cls"), - ("running", "cls"), - ("completed", "cls"), - ("get_log_level", "get"), - ("get_provider", "get"), - ("process_chunk", "process"), - ("get_openai_client", "_get_openai_client"), - ("meeting_apps", "frozenset"), - ("get_sync_run", "get"), - ("list_all", "list"), - ("get_by_id", "get"), - ("check_otel_available", "_check_otel_available"), - ("start_span", "_NoOpSpan"), - ("detected_app", "next"), - } - - violations: list[Violation] = [] parse_errors: list[str] = [] - - for py_file in find_source_files(): - tree, error = parse_file_safe(py_file) - if error: - parse_errors.append(error) - continue - if tree is None: - continue - - rel_path = relative_path(py_file) - if "converters" in rel_path: - continue - - for node in ast.walk(tree): - if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)): - if node.name.startswith("_"): - continue - - wrapped = is_thin_wrapper(node) - if wrapped and node.name != wrapped: - # Skip known valid patterns - if (node.name, wrapped) in allowed_wrappers: - continue - violations.append( - Violation( - rule="thin_wrapper", - relative_path=rel_path, - identifier=node.name, - detail=wrapped, - ) - ) + violations = collect_thin_wrappers(parse_errors=parse_errors) collect_parse_errors(parse_errors) assert_no_new_violations("thin_wrapper", violations) @@ -141,69 +41,15 @@ def test_no_trivial_wrapper_functions() -> None: def test_no_alias_imports() -> None: """Detect imports that alias to confusing names.""" - alias_pattern = re.compile(r"^import\s+(\w+)\s+as\s+(\w+)") - from_alias_pattern = re.compile(r"from\s+\S+\s+import\s+(\w+)\s+as\s+(\w+)") - - violations: list[Violation] = [] - - for py_file in find_source_files(): - content, error = read_file_safe(py_file) - if error or content is None: - continue - - rel_path = relative_path(py_file) - lines = content.splitlines() - - for i, line in enumerate(lines, start=1): - for pattern in [alias_pattern, from_alias_pattern]: - if match := pattern.search(line): - original, alias = match.groups() - if original.lower() not in alias.lower(): - # Common well-known aliases that don't need original name - if alias not in {"np", "pd", "plt", "tf", "nn", "F", "sa", "sd"}: - violations.append( - Violation( - rule="alias_import", - relative_path=rel_path, - identifier=content_hash(f"{i}:{line.strip()}"), - detail=f"{original}->{alias}", - ) - ) + violations = collect_alias_imports() assert_no_new_violations("alias_import", violations) def test_no_redundant_type_aliases() -> None: """Detect type aliases that don't add semantic meaning.""" - violations: list[Violation] = [] parse_errors: list[str] = [] - - for py_file in find_source_files(): - tree, error = parse_file_safe(py_file) - if error: - parse_errors.append(error) - continue - if tree is None: - continue - - rel_path = relative_path(py_file) - - for node in ast.walk(tree): - if isinstance(node, ast.AnnAssign) and isinstance(node.target, ast.Name): - if isinstance(node.annotation, ast.Name): - if node.annotation.id == "TypeAlias": - if isinstance(node.value, ast.Name): - base_type = node.value.id - if base_type in {"str", "int", "float", "bool", "bytes"}: - target_name = node.target.id - violations.append( - Violation( - rule="redundant_type_alias", - relative_path=rel_path, - identifier=target_name, - detail=base_type, - ) - ) + violations = collect_redundant_type_aliases(parse_errors=parse_errors) collect_parse_errors(parse_errors) assert_no_new_violations("redundant_type_alias", violations) @@ -211,49 +57,8 @@ def test_no_redundant_type_aliases() -> None: def test_no_passthrough_classes() -> None: """Detect classes that only delegate to another object.""" - # Classes that are intentionally factory-pattern based (all methods return cls()) - allowed_factory_classes: set[str] = set() - - violations: list[Violation] = [] parse_errors: list[str] = [] - - for py_file in find_source_files(): - tree, error = parse_file_safe(py_file) - if error: - parse_errors.append(error) - continue - if tree is None: - continue - - rel_path = relative_path(py_file) - - for node in ast.walk(tree): - if isinstance(node, ast.ClassDef): - # Skip allowed factory classes - if node.name in allowed_factory_classes: - continue - - methods = [ - n - for n in node.body - if isinstance(n, (ast.FunctionDef, ast.AsyncFunctionDef)) - and not n.name.startswith("_") - ] - - if len(methods) >= 3: - passthrough_count = sum( - 1 for m in methods if is_thin_wrapper(m) is not None - ) - - if passthrough_count == len(methods): - violations.append( - Violation( - rule="passthrough_class", - relative_path=rel_path, - identifier=node.name, - detail=f"{passthrough_count}_methods", - ) - ) + violations = collect_passthrough_classes(parse_errors=parse_errors) collect_parse_errors(parse_errors) assert_no_new_violations("passthrough_class", violations) diff --git a/tests/test_server_address_sync.py b/tests/test_server_address_sync.py index 3342136..ce46c34 100644 --- a/tests/test_server_address_sync.py +++ b/tests/test_server_address_sync.py @@ -16,7 +16,7 @@ from pathlib import Path import pytest -from noteflow.grpc._config import DEFAULT_BIND_ADDRESS +from noteflow.grpc.config.config import DEFAULT_BIND_ADDRESS # Path to Rust config file RUST_CONFIG_PATH = Path(__file__).parent.parent / "client/src-tauri/src/config.rs"