From 3592d6c63ecfb39deb2a1227d992641e520137c6 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 30 Apr 2026 16:48:37 +0000 Subject: [PATCH 1/2] feat: add automated test suite, GitHub Actions CI, and CONTRIBUTING.md - Add tests/ directory with 133 pytest tests covering: - batch_creation: JSONL generation, forbid_additional_props, prompt construction - batch_parsing: _safe_parse_model_text, get_client, get_batch_results with mocked API - data_conversion: make_str_enum, to_long_df, join_datasets - data_import: CSV/TSV/Excel reading without GUI - settings: user config load/save/precedence - reliability: compute_cohens_kappa pure logic - Add tests/fixtures/ with sample CSV data and a realistic fixture - Add requirements-dev.txt (pytest, pytest-mock) - Add pytest.ini configuration - Add .github/workflows/tests.yml (CI on push + PR, Python 3.10/3.11/3.12) - Add CONTRIBUTING.md with setup, testing, structure, and contribution guide - Update README.md with developer/testing section Agent-Logs-Url: https://github.com/tmaier-kettering/CodebookAI/sessions/e942e5e4-04d0-4469-bc8e-c6be9e3afdaa Co-authored-by: tmaier-kettering <109093855+tmaier-kettering@users.noreply.github.com> --- .github/workflows/tests.yml | 48 ++++ CONTRIBUTING.md | 234 ++++++++++++++++++ README.md | 25 ++ pytest.ini | 6 + requirements-dev.txt | 5 + tests/__init__.py | 0 tests/conftest.py | 98 ++++++++ tests/fixtures/realistic_dataset.csv | 6 + tests/fixtures/sample_labels.csv | 4 + tests/fixtures/sample_quotes.csv | 4 + tests/test_batch_creation.py | 262 +++++++++++++++++++++ tests/test_batch_parsing.py | 340 +++++++++++++++++++++++++++ tests/test_data_conversion.py | 179 ++++++++++++++ tests/test_data_import.py | 177 ++++++++++++++ tests/test_reliability.py | 86 +++++++ tests/test_settings.py | 172 ++++++++++++++ 16 files changed, 1646 insertions(+) create mode 100644 .github/workflows/tests.yml create mode 100644 CONTRIBUTING.md create mode 100644 pytest.ini create mode 100644 requirements-dev.txt create mode 100644 tests/__init__.py create mode 100644 tests/conftest.py create mode 100644 tests/fixtures/realistic_dataset.csv create mode 100644 tests/fixtures/sample_labels.csv create mode 100644 tests/fixtures/sample_quotes.csv create mode 100644 tests/test_batch_creation.py create mode 100644 tests/test_batch_parsing.py create mode 100644 tests/test_data_conversion.py create mode 100644 tests/test_data_import.py create mode 100644 tests/test_reliability.py create mode 100644 tests/test_settings.py diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml new file mode 100644 index 0000000..6f2e4cb --- /dev/null +++ b/.github/workflows/tests.yml @@ -0,0 +1,48 @@ +name: Tests + +on: + push: + branches: ["**"] + pull_request: + branches: ["**"] + +jobs: + test: + name: Test (Python ${{ matrix.python-version }}) + runs-on: ubuntu-latest + + strategy: + fail-fast: false + matrix: + python-version: ["3.10", "3.11", "3.12"] + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + + - name: Install system dependencies (tkinter + Xvfb) + run: | + sudo apt-get update -qq + sudo apt-get install -y python3-tk xvfb + + - name: Install Python dependencies + run: | + python -m pip install --upgrade pip + pip install -r requirements.txt + pip install -r requirements-dev.txt + + - name: Run test suite + # xvfb-run ensures a virtual display is available for any code that + # needs one (e.g. tkinter internals on import), even though our tests + # do not open windows. + run: xvfb-run -a pytest tests/ -v --tb=short + env: + MPLBACKEND: Agg + # Prevent keyring from trying to connect to GNOME Keyring / KWallet; + # the test suite installs its own in-memory backend via conftest.py. + PYTHON_KEYRING_BACKEND: keyring.backends.null.Keyring diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..627759d --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,234 @@ +# Contributing to CodebookAI + +Thank you for your interest in contributing to CodebookAI! This guide explains +how to set up your development environment, run the test suite, and submit +changes. + +--- + +## Table of Contents + +1. [Getting Started](#getting-started) +2. [Running the Application Locally](#running-the-application-locally) +3. [Running the Tests](#running-the-tests) +4. [Project Structure](#project-structure) +5. [Coding Conventions](#coding-conventions) +6. [Reporting Bugs](#reporting-bugs) +7. [Proposing Changes](#proposing-changes) +8. [Getting Support](#getting-support) + +--- + +## Getting Started + +### Prerequisites + +| Requirement | Version | +|-------------|---------| +| Python | 3.10 or later | +| tkinter | bundled with most Python installers; see note below | + +> **Linux note:** tkinter is a separate package on many distros. +> Install it with: +> ```bash +> # Ubuntu / Debian +> sudo apt-get install python3-tk +> # Fedora / RHEL +> sudo dnf install python3-tkinter +> ``` + +### Clone and install dependencies + +```bash +git clone https://github.com/tmaier-kettering/CodebookAI.git +cd CodebookAI + +# Create and activate a virtual environment (recommended) +python -m venv .venv +source .venv/bin/activate # macOS / Linux +.venv\Scripts\activate # Windows PowerShell + +# Install runtime dependencies +pip install -r requirements.txt + +# Install development/testing dependencies +pip install -r requirements-dev.txt +``` + +--- + +## Running the Application Locally + +```bash +python main.py +``` + +The application requires an OpenAI API key. On first launch, go to +**File → Settings** and paste your key. The key is stored in your OS +credential vault (Windows Credential Manager / macOS Keychain / Linux +Secret Service) and is never written to disk in plain text. + +--- + +## Running the Tests + +The test suite uses [pytest](https://docs.pytest.org/) and covers the +application's core logic without requiring a live OpenAI API key or a +graphical display. + +```bash +# Run the full test suite +pytest + +# Run a specific file +pytest tests/test_batch_creation.py + +# Run a specific test class or function +pytest tests/test_batch_parsing.py::TestSafeParseModelText +pytest tests/test_batch_parsing.py::TestSafeParseModelText::test_valid_json_returns_dict_no_error + +# Show verbose output and full tracebacks +pytest -v --tb=long +``` + +### What the tests cover + +| File | Coverage area | +|------|---------------| +| `test_batch_creation.py` | JSONL batch generation — prompt construction, schema building, `forbid_additional_props`, non-ASCII text, edge cases | +| `test_batch_parsing.py` | Result parsing (`_safe_parse_model_text`), OpenAI client creation, `get_batch_results` with mocked API (success, auth failure, rate limit, timeout, malformed output, fenced JSON) | +| `test_data_conversion.py` | `make_str_enum`, `to_long_df` (including multi-label explode), `join_datasets` | +| `test_data_import.py` | Delimiter sniffing, CSV/TSV/Excel reading, non-ASCII content, empty files, realistic fixture | +| `test_settings.py` | User config load/save/roundtrip, `get_setting` precedence | +| `test_reliability.py` | Cohen's kappa — perfect agreement, known value, edge cases (empty, mismatched lengths, single label, unicode labels) | + +### Headless / CI environments + +The tests do **not** open any GUI windows, so they run fine in headless +environments (GitHub Actions, Docker containers, SSH sessions). If you +hit display-related errors, wrap the pytest invocation with `xvfb-run -a`: + +```bash +sudo apt-get install -y xvfb +xvfb-run -a pytest +``` + +The project's GitHub Actions workflow does this automatically. + +--- + +## Project Structure + +``` +CodebookAI/ +├── main.py Entry point – creates the Tkinter root window +├── requirements.txt Runtime dependencies +├── requirements-dev.txt Test / development dependencies +├── pytest.ini Pytest configuration +│ +├── batch_processing/ OpenAI Batch API workflow +│ ├── batch_creation.py JSONL generation (pure, fully testable) +│ ├── batch_method.py Batch submission / retrieval / result parsing +│ └── batch_error_handling.py Error reporting UI +│ +├── live_processing/ Real-time classification +│ ├── single_label_live.py Single-label pipeline +│ ├── multi_label_live.py Multi-label pipeline +│ ├── reliability_calculator.py Cohen's kappa (pure, fully testable) +│ ├── keyword_extraction_live.py +│ ├── correlogram.py +│ └── sampler.py +│ +├── file_handling/ File I/O (mostly pure, fully testable) +│ ├── data_import.py CSV / Excel reader + import-wizard GUI +│ └── data_conversion.py make_str_enum, to_long_df, join_datasets +│ +├── settings/ Configuration +│ ├── config.py Default constants +│ ├── user_config.py JSON-based user settings (fully testable) +│ ├── secrets_store.py OS keyring wrapper +│ └── models_registry.py OpenAI model list cache +│ +├── ui/ Tkinter GUI components +│ +├── tests/ Automated test suite +│ ├── conftest.py Shared fixtures; in-memory keyring setup +│ ├── fixtures/ Static data files used by tests +│ │ ├── sample_labels.csv +│ │ ├── sample_quotes.csv +│ │ └── realistic_dataset.csv +│ ├── test_batch_creation.py +│ ├── test_batch_parsing.py +│ ├── test_data_conversion.py +│ ├── test_data_import.py +│ ├── test_settings.py +│ └── test_reliability.py +│ +├── example_dataset/ Bundled example data +├── assets/ Images and icons +└── wiki/ Documentation +``` + +--- + +## Coding Conventions + +* **Python 3.10+** — the codebase uses `X | Y` union syntax in type annotations. +* **Type annotations** are used throughout; please annotate new public functions + and classes. +* **Pydantic v2** is used for data validation; follow the existing + `model_config = ConfigDict(...)` pattern. +* **No external side effects at module level** — avoid making network or file + system calls at module import time. (The existing `models_registry.py` is a + known exception that is guarded with `try/except`.) +* Keep GUI code (tkinter) in the `ui/` package or in thin UI-layer functions. + Business logic and data processing should be in separate, GUI-free modules + so they can be unit-tested without a display. +* Match the docstring style already present in the file you are editing. + +--- + +## Reporting Bugs + +1. Check the [existing issues](https://github.com/tmaier-kettering/CodebookAI/issues) + to see if the problem has already been reported. +2. If not, open a new issue and include: + - A clear title and description of the problem. + - Steps to reproduce, including the input data if applicable. + - The Python version, OS, and CodebookAI version (or git commit hash). + - Any error messages or stack traces from the console. + +--- + +## Proposing Changes + +1. **Fork** the repository and create a feature branch: + ```bash + git checkout -b feature/my-improvement + ``` +2. Make your changes and **add or update tests** in the `tests/` directory. +3. Verify the full test suite passes: + ```bash + pytest + ``` +4. Open a **pull request** against the `main` branch with a clear description + of what was changed and why. + +We use GitHub Actions to run the test suite automatically on every pull request. +A passing CI run is required before a PR can be merged. + +--- + +## Getting Support + +* **Documentation:** see the [wiki folder](./wiki/) for user guides. +* **Questions or ideas:** open a [GitHub Discussion](https://github.com/tmaier-kettering/CodebookAI/discussions) + or a [GitHub Issue](https://github.com/tmaier-kettering/CodebookAI/issues). +* **Security vulnerabilities:** please report them privately via GitHub's + [security advisory](https://github.com/tmaier-kettering/CodebookAI/security/advisories) + feature rather than in a public issue. + +--- + +*CodebookAI is an MIT-licensed open-source project. Contributions of all kinds +are welcome.* diff --git a/README.md b/README.md index 609658b..3caf2ae 100644 --- a/README.md +++ b/README.md @@ -20,6 +20,31 @@ CodebookAI is a tool designed to assist qualitative researchers in processing la --- +## Developer Guide + +### Running Tests + +The repository includes an automated test suite that runs without a live +OpenAI API key or a graphical display. + +```bash +# Install dependencies +pip install -r requirements.txt +pip install -r requirements-dev.txt + +# Run all tests +pytest +``` + +Continuous integration is configured via GitHub Actions (`.github/workflows/tests.yml`) +and runs on every push and pull request across Python 3.10, 3.11, and 3.12. + +See [CONTRIBUTING.md](CONTRIBUTING.md) for a full guide on setting up a local +development environment, the project structure, coding conventions, and how to +submit changes. + +--- + ## How to Support [![BuyMeACoffee](./assets/buymeacoffee.png)](https://buymeacoffee.com/professthor) diff --git a/pytest.ini b/pytest.ini new file mode 100644 index 0000000..9855d94 --- /dev/null +++ b/pytest.ini @@ -0,0 +1,6 @@ +[pytest] +testpaths = tests +python_files = test_*.py +python_classes = Test* +python_functions = test_* +addopts = -v --tb=short diff --git a/requirements-dev.txt b/requirements-dev.txt new file mode 100644 index 0000000..12d4363 --- /dev/null +++ b/requirements-dev.txt @@ -0,0 +1,5 @@ +# Development / testing dependencies +# Install with: pip install -r requirements-dev.txt + +pytest>=7.4.0 # Test runner +pytest-mock>=3.12.0 # Convenient mocker fixture (wraps unittest.mock) diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..9815035 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,98 @@ +""" +Shared pytest configuration, fixtures, and helpers for the CodebookAI test suite. + +Sets up: +- The project root on sys.path so tests can import application modules. +- An in-memory keyring backend so tests never touch the OS credential store. +- MPLBACKEND=Agg so matplotlib never tries to open a GUI window. +""" + +import os +import sys + +# --------------------------------------------------------------------------- +# Environment must be configured BEFORE any project module is imported. +# --------------------------------------------------------------------------- +os.environ.setdefault("MPLBACKEND", "Agg") + +# Ensure the project root is importable as top-level packages. +_PROJECT_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +if _PROJECT_ROOT not in sys.path: + sys.path.insert(0, _PROJECT_ROOT) + +# --------------------------------------------------------------------------- +# Install an in-memory keyring backend before any project code is imported. +# This prevents tests from touching the OS credential store (GNOME Keyring, +# Windows Credential Manager, etc.) and makes keyring behaviour deterministic. +# --------------------------------------------------------------------------- +import keyring +from keyring.backend import KeyringBackend + + +class _InMemoryKeyring(KeyringBackend): + """Thread-safe in-memory keyring used exclusively during tests.""" + + priority = 999 # always wins the priority contest + + def __init__(self): + self._store: dict = {} + + def set_password(self, service: str, username: str, password: str) -> None: + self._store[(service, username)] = password + + def get_password(self, service: str, username: str): + return self._store.get((service, username)) + + def delete_password(self, service: str, username: str) -> None: + self._store.pop((service, username), None) + + +keyring.set_keyring(_InMemoryKeyring()) + +# --------------------------------------------------------------------------- +# Pytest fixtures +# --------------------------------------------------------------------------- +import pytest +from pathlib import Path + +FIXTURES_DIR = Path(__file__).parent / "fixtures" + + +@pytest.fixture +def sample_labels(): + """Return a small list of sentiment labels matching the fixture CSV.""" + return ["positive", "negative", "neutral"] + + +@pytest.fixture +def sample_quotes(): + """Return a small list of text excerpts for classification tests.""" + return [ + "The product exceeded all of my expectations.", + "This was a deeply disappointing experience.", + "The item arrived on time and was as described.", + ] + + +@pytest.fixture +def realistic_quotes(): + """A slightly richer set that includes non-ASCII and longer excerpts.""" + return [ + "For the last quarter of 2010, net sales doubled to EUR131m.", + "All the major construction companies of Finland are operating in Russia.", + "The Company updates its full year outlook and estimates its results to remain at loss.", + "Net sales rose by some 14 % year-on-year in the first nine months.", + "The café déjà vu — naïve résumé — piñata fiancée.", + ] + + +@pytest.fixture +def tmp_csv(tmp_path): + """Return a factory that writes a CSV string to a temp file and returns its path.""" + + def _make(content: str, filename: str = "test.csv") -> Path: + p = tmp_path / filename + p.write_text(content, encoding="utf-8") + return p + + return _make diff --git a/tests/fixtures/realistic_dataset.csv b/tests/fixtures/realistic_dataset.csv new file mode 100644 index 0000000..426a94c --- /dev/null +++ b/tests/fixtures/realistic_dataset.csv @@ -0,0 +1,6 @@ +text,id,emotion +"For the last quarter of 2010, Componenta's net sales doubled to EUR131m from EUR76m for the same period a year earlier, while it moved to a zero pre-tax profit from a pre-tax loss of EUR7m.",1,positive +"According to the Finnish-Russian Chamber of Commerce, all the major construction companies of Finland are operating in Russia.",2,neutral +"SSH COMMUNICATIONS SECURITY CORP STOCK EXCHANGE RELEASE OCTOBER 14, 2008 AT 2:45 PM The Company updates its full year outlook and estimates its results to remain at loss for the full year.",3,negative +"Kone's net sales rose by some 14 % year-on-year in the first nine months of 2008.",4,positive +"The café déjà vu — naïve résumé — piñata fiancée.",5,neutral diff --git a/tests/fixtures/sample_labels.csv b/tests/fixtures/sample_labels.csv new file mode 100644 index 0000000..a9802e7 --- /dev/null +++ b/tests/fixtures/sample_labels.csv @@ -0,0 +1,4 @@ +label +positive +negative +neutral diff --git a/tests/fixtures/sample_quotes.csv b/tests/fixtures/sample_quotes.csv new file mode 100644 index 0000000..ffc8ea8 --- /dev/null +++ b/tests/fixtures/sample_quotes.csv @@ -0,0 +1,4 @@ +text +The product exceeded all of my expectations and I would buy it again. +This was a deeply disappointing experience and I want a refund. +The item arrived on time and was exactly as described. diff --git a/tests/test_batch_creation.py b/tests/test_batch_creation.py new file mode 100644 index 0000000..5fd0b11 --- /dev/null +++ b/tests/test_batch_creation.py @@ -0,0 +1,262 @@ +""" +Tests for batch_processing/batch_creation.py + +Covers: +- forbid_additional_props schema transformation +- generate_single_label_batch: JSONL output, prompt content, custom_id format, schema +- generate_multi_label_batch: JSONL output and array schema +- generate_keyword_extraction_batch: JSONL output +- Edge cases: empty input, non-ASCII text, punctuation-heavy text +""" + +import io +import json + +import pytest + +from batch_processing.batch_creation import ( + forbid_additional_props, + generate_single_label_batch, + generate_multi_label_batch, + generate_keyword_extraction_batch, +) + + +# --------------------------------------------------------------------------- +# forbid_additional_props +# --------------------------------------------------------------------------- + +class TestForbidAdditionalProps: + def test_sets_flag_on_object(self): + schema = {"type": "object", "properties": {"x": {"type": "string"}}} + result = forbid_additional_props(schema) + assert result["additionalProperties"] is False + + def test_recursive_nested_object(self): + schema = { + "type": "object", + "properties": { + "inner": { + "type": "object", + "properties": {"y": {"type": "integer"}}, + } + }, + } + result = forbid_additional_props(schema) + assert result["additionalProperties"] is False + assert result["properties"]["inner"]["additionalProperties"] is False + + def test_array_items_object(self): + schema = { + "type": "array", + "items": { + "type": "object", + "properties": {"z": {"type": "string"}}, + }, + } + result = forbid_additional_props(schema) + assert result["items"]["additionalProperties"] is False + + def test_non_object_schema_unchanged(self): + schema = {"type": "string"} + result = forbid_additional_props(schema) + assert "additionalProperties" not in result + + def test_creates_empty_properties_on_object(self): + schema = {"type": "object"} + result = forbid_additional_props(schema) + assert result["additionalProperties"] is False + assert "properties" in result + + def test_does_not_modify_string_inside_array(self): + schema = {"type": "array", "items": {"type": "string"}} + result = forbid_additional_props(schema) + assert "additionalProperties" not in result["items"] + + def test_handles_non_dict_gracefully(self): + # Passing a non-dict should be returned unchanged + assert forbid_additional_props("string") == "string" + assert forbid_additional_props(42) == 42 + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + +def _parse_jsonl(buf: io.BytesIO) -> list[dict]: + """Read all lines from a BytesIO and parse them as JSON.""" + buf.seek(0) + return [json.loads(line) for line in buf.read().decode("utf-8").splitlines() if line.strip()] + + +# --------------------------------------------------------------------------- +# generate_single_label_batch +# --------------------------------------------------------------------------- + +class TestGenerateSingleLabelBatch: + def test_returns_bytesio(self, sample_labels, sample_quotes): + result = generate_single_label_batch(sample_labels, sample_quotes) + assert isinstance(result, io.BytesIO) + + def test_has_jsonl_name(self, sample_labels, sample_quotes): + result = generate_single_label_batch(sample_labels, sample_quotes) + assert result.name == "batchinput.jsonl" + + def test_line_count_matches_quotes(self, sample_labels, sample_quotes): + result = generate_single_label_batch(sample_labels, sample_quotes) + lines = _parse_jsonl(result) + assert len(lines) == len(sample_quotes) + + def test_custom_id_format(self, sample_labels, sample_quotes): + result = generate_single_label_batch(sample_labels, sample_quotes) + lines = _parse_jsonl(result) + for i, line in enumerate(lines, start=1): + assert line["custom_id"] == f"quote-{i:05d}", f"Wrong custom_id at index {i}" + + def test_prompt_contains_all_labels(self, sample_labels, sample_quotes): + result = generate_single_label_batch(sample_labels, sample_quotes) + lines = _parse_jsonl(result) + for line in lines: + content = line["body"]["input"][0]["content"] + for label in sample_labels: + assert label in content, f"Label '{label}' missing from prompt" + + def test_prompt_contains_quote_text(self, sample_labels, sample_quotes): + result = generate_single_label_batch(sample_labels, sample_quotes) + lines = _parse_jsonl(result) + for line, quote in zip(lines, sample_quotes): + assert quote in line["body"]["input"][0]["content"] + + def test_schema_strict_flag(self, sample_labels, sample_quotes): + result = generate_single_label_batch(sample_labels, sample_quotes) + lines = _parse_jsonl(result) + for line in lines: + fmt = line["body"]["text"]["format"] + assert fmt["strict"] is True + + def test_schema_additional_props_false(self, sample_labels, sample_quotes): + result = generate_single_label_batch(sample_labels, sample_quotes) + lines = _parse_jsonl(result) + for line in lines: + schema = line["body"]["text"]["format"]["schema"] + assert schema.get("additionalProperties") is False + + def test_metadata_includes_quote(self, sample_labels, sample_quotes): + result = generate_single_label_batch(sample_labels, sample_quotes) + lines = _parse_jsonl(result) + for line, quote in zip(lines, sample_quotes): + assert line["body"]["metadata"]["quote"] == quote + + def test_empty_quotes_returns_empty_jsonl(self, sample_labels): + result = generate_single_label_batch(sample_labels, []) + lines = _parse_jsonl(result) + assert lines == [] + + def test_nonascii_text_round_trips(self, sample_labels): + quotes = ["Ünïcödé tëxt wïth spëcïäl châräctërs!", "日本語テキスト", "Héllo wörld"] + result = generate_single_label_batch(sample_labels, quotes) + lines = _parse_jsonl(result) + assert len(lines) == 3 + assert "日本語テキスト" in lines[1]["body"]["input"][0]["content"] + + def test_punctuation_heavy_text(self, sample_labels): + quotes = ['He said, "It\'s great!" — really? (Yes!) #amazing @user $100'] + result = generate_single_label_batch(sample_labels, quotes) + lines = _parse_jsonl(result) + assert len(lines) == 1 + + def test_long_excerpt(self, sample_labels): + long_quote = "This is a very long excerpt. " * 100 + result = generate_single_label_batch(sample_labels, [long_quote]) + lines = _parse_jsonl(result) + assert len(lines) == 1 + + def test_short_single_word_excerpt(self, sample_labels): + result = generate_single_label_batch(sample_labels, ["great"]) + lines = _parse_jsonl(result) + assert len(lines) == 1 + + def test_url_endpoint_is_responses(self, sample_labels, sample_quotes): + result = generate_single_label_batch(sample_labels, sample_quotes) + lines = _parse_jsonl(result) + for line in lines: + assert line["url"] == "/v1/responses" + assert line["method"] == "POST" + + +# --------------------------------------------------------------------------- +# generate_multi_label_batch +# --------------------------------------------------------------------------- + +class TestGenerateMultiLabelBatch: + def test_line_count_matches_quotes(self, sample_labels, sample_quotes): + result = generate_multi_label_batch(sample_labels, sample_quotes) + lines = _parse_jsonl(result) + assert len(lines) == len(sample_quotes) + + def test_schema_label_field_is_array(self, sample_labels, sample_quotes): + result = generate_multi_label_batch(sample_labels, sample_quotes) + lines = _parse_jsonl(result) + for line in lines: + schema = line["body"]["text"]["format"]["schema"] + label_prop = schema["properties"]["label"] + assert label_prop["type"] == "array" + + def test_prompt_contains_all_labels(self, sample_labels, sample_quotes): + result = generate_multi_label_batch(sample_labels, sample_quotes) + lines = _parse_jsonl(result) + for line in lines: + content = line["body"]["input"][0]["content"] + for label in sample_labels: + assert label in content + + def test_empty_quotes(self, sample_labels): + result = generate_multi_label_batch(sample_labels, []) + assert _parse_jsonl(result) == [] + + def test_nonascii_labels(self, sample_quotes): + labels = ["positivo", "négative", "中立"] + result = generate_multi_label_batch(labels, sample_quotes) + lines = _parse_jsonl(result) + assert len(lines) == len(sample_quotes) + for line in lines: + content = line["body"]["input"][0]["content"] + assert "中立" in content + + +# --------------------------------------------------------------------------- +# generate_keyword_extraction_batch +# --------------------------------------------------------------------------- + +class TestGenerateKeywordExtractionBatch: + def test_line_count_matches_texts(self, sample_quotes): + result = generate_keyword_extraction_batch(sample_quotes) + lines = _parse_jsonl(result) + assert len(lines) == len(sample_quotes) + + def test_custom_id_prefix_is_text(self, sample_quotes): + result = generate_keyword_extraction_batch(sample_quotes) + lines = _parse_jsonl(result) + for i, line in enumerate(lines, start=1): + assert line["custom_id"] == f"text-{i:05d}" + + def test_schema_has_keywords_array(self, sample_quotes): + result = generate_keyword_extraction_batch(sample_quotes) + lines = _parse_jsonl(result) + for line in lines: + schema = line["body"]["text"]["format"]["schema"] + assert "keywords" in schema.get("properties", {}) + assert schema["properties"]["keywords"]["type"] == "array" + + def test_prompt_contains_text(self, sample_quotes): + result = generate_keyword_extraction_batch(sample_quotes) + lines = _parse_jsonl(result) + for line, text in zip(lines, sample_quotes): + user_msg = next( + m for m in line["body"]["input"] if m["role"] == "user" + ) + assert text in user_msg["content"] + + def test_empty_texts(self): + result = generate_keyword_extraction_batch([]) + assert _parse_jsonl(result) == [] diff --git a/tests/test_batch_parsing.py b/tests/test_batch_parsing.py new file mode 100644 index 0000000..92d78fa --- /dev/null +++ b/tests/test_batch_parsing.py @@ -0,0 +1,340 @@ +""" +Tests for batch_processing/batch_method.py + +Covers: +- _safe_parse_model_text: valid JSON, fenced JSON, truncated JSON, non-JSON, + completely unparsable, empty string, whitespace +- get_client: missing API key raises, valid key returns client +- get_batch_results: successful response, malformed rows, missing output file +- Authentication failures, rate limit errors, timeout/connection errors are + all handled via the same Exception path that get_batch_results delegates to + the caller (get_client raises, or the mock client raises). +""" + +import json +from unittest.mock import MagicMock + +import pytest + +from batch_processing.batch_method import ( + _safe_parse_model_text, + get_batch_results, + get_client, +) + + +# --------------------------------------------------------------------------- +# _safe_parse_model_text +# --------------------------------------------------------------------------- + +class TestSafeParseModelText: + """Tests for the internal JSON-parsing/repair helper.""" + + def test_valid_json_returns_dict_no_error(self): + result, err = _safe_parse_model_text('{"label": "positive"}') + assert result == {"label": "positive"} + assert err is None + + def test_valid_json_multi_key(self): + result, err = _safe_parse_model_text('{"label": "positive", "confidence": 0.9}') + assert result["label"] == "positive" + assert err is None + + def test_whitespace_around_json(self): + result, err = _safe_parse_model_text(' \n{"label": "neutral"}\n ') + assert result == {"label": "neutral"} + assert err is None + + def test_fenced_json_single_backtick_block(self): + text = '```json\n{"label": "negative"}\n```' + result, err = _safe_parse_model_text(text) + assert result == {"label": "negative"} + assert err is None + + def test_fenced_json_no_language_tag(self): + text = '```\n{"label": "positive"}\n```' + result, err = _safe_parse_model_text(text) + assert result == {"label": "positive"} + assert err is None + + def test_truncated_json_brace_label(self): + """Truncated '{"label":"disapproval' should be repaired.""" + result, err = _safe_parse_model_text('{"label":"negative') + assert result is not None + assert result.get("label") == "negative" + assert err is not None # repair note present + + def test_non_json_label_colon_format(self): + """'label: positive' should be recovered.""" + result, err = _safe_parse_model_text('label: positive') + assert result is not None + assert result.get("label") == "positive" + assert err is not None + + def test_non_json_label_equals_format(self): + result, err = _safe_parse_model_text('label = "negative"') + assert result is not None + assert result.get("label") == "negative" + assert err is not None + + def test_completely_unparsable_returns_none(self): + result, err = _safe_parse_model_text("COMPLETELY RANDOM UNSTRUCTURED OUTPUT XYZ") + assert result is None + assert err is not None + + def test_empty_string_returns_none(self): + result, err = _safe_parse_model_text("") + assert result is None + assert err is not None + + def test_whitespace_only_returns_none(self): + result, err = _safe_parse_model_text(" \n\t ") + assert result is None + assert err is not None + + def test_json_array_returns_parsed(self): + """A JSON array is valid JSON and should be parsed as-is.""" + result, err = _safe_parse_model_text('["positive", "negative"]') + assert result == ["positive", "negative"] + assert err is None + + def test_truncated_label_with_spaces(self): + """Truncation mid-value with a space in the value.""" + result, err = _safe_parse_model_text('{"label":"strongly positive') + assert result is not None + assert "label" in result + + +# --------------------------------------------------------------------------- +# get_client +# --------------------------------------------------------------------------- + +class TestGetClient: + def test_raises_when_no_api_key(self, mocker): + mocker.patch( + "batch_processing.batch_method.secrets_store.load_api_key", + return_value=None, + ) + with pytest.raises(Exception, match="API key not configured"): + get_client() + + def test_returns_openai_client_when_key_present(self, mocker): + mocker.patch( + "batch_processing.batch_method.secrets_store.load_api_key", + return_value="sk-test-fake-key", + ) + mock_openai_cls = mocker.patch("batch_processing.batch_method.OpenAI") + client = get_client() + mock_openai_cls.assert_called_once_with(api_key="sk-test-fake-key") + assert client is mock_openai_cls.return_value + + +# --------------------------------------------------------------------------- +# get_batch_results – mocked OpenAI client +# --------------------------------------------------------------------------- + +def _make_response_line(custom_id: str, text_output: str, quote: str = "") -> str: + """Build a single JSONL line matching the OpenAI batch output format.""" + return json.dumps( + { + "custom_id": custom_id, + "response": { + "body": { + "output": [{"content": [{"text": text_output}]}], + "metadata": {"quote": quote}, + } + }, + } + ) + + +class TestGetBatchResults: + def _setup_mock_client(self, mocker, output_file_id, file_content_bytes): + mock_client = MagicMock() + mocker.patch( + "batch_processing.batch_method.get_client", return_value=mock_client + ) + mock_status = MagicMock() + mock_status.output_file_id = output_file_id + mock_client.batches.retrieve.return_value = mock_status + mock_client.files.content.return_value.content = file_content_bytes + return mock_client, mock_status + + def test_successful_single_row(self, mocker): + line = _make_response_line( + "quote-00001", '{"label": "positive"}', quote="Great product!" + ) + mock_client, _ = self._setup_mock_client( + mocker, "file-123", (line + "\n").encode("utf-8") + ) + mock_save = mocker.patch("batch_processing.batch_method.save_as_csv") + + get_batch_results("batch-001") + + mock_save.assert_called_once() + df = mock_save.call_args[0][0] + assert "label" in df.columns + assert df["label"].iloc[0] == "positive" + + def test_multiple_rows_all_parsed(self, mocker): + lines = "\n".join( + [ + _make_response_line("quote-00001", '{"label": "positive"}', "Good."), + _make_response_line("quote-00002", '{"label": "negative"}', "Bad."), + _make_response_line("quote-00003", '{"label": "neutral"}', "OK."), + ] + ) + self._setup_mock_client(mocker, "file-234", (lines + "\n").encode("utf-8")) + mock_save = mocker.patch("batch_processing.batch_method.save_as_csv") + + get_batch_results("batch-002") + + df = mock_save.call_args[0][0] + assert len(df) == 3 + assert list(df["label"]) == ["positive", "negative", "neutral"] + + def test_malformed_row_excluded_from_results(self, mocker): + """A row whose text cannot be parsed should be skipped (bad_rows), not crash.""" + bad_line = _make_response_line( + "quote-00001", "COMPLETELY INVALID OUTPUT", "Some quote." + ) + self._setup_mock_client( + mocker, "file-345", (bad_line + "\n").encode("utf-8") + ) + mock_save = mocker.patch("batch_processing.batch_method.save_as_csv") + + get_batch_results("batch-003") + + df = mock_save.call_args[0][0] + assert len(df) == 0 + + def test_mixed_good_and_bad_rows(self, mocker): + """Only parsable rows end up in the saved DataFrame.""" + lines = "\n".join( + [ + _make_response_line("quote-00001", '{"label": "positive"}', "Good."), + _make_response_line("quote-00002", "NOT JSON AT ALL", "Bad."), + _make_response_line("quote-00003", '{"label": "neutral"}', "OK."), + ] + ) + self._setup_mock_client(mocker, "file-456", (lines + "\n").encode("utf-8")) + mock_save = mocker.patch("batch_processing.batch_method.save_as_csv") + + get_batch_results("batch-004") + + df = mock_save.call_args[0][0] + assert len(df) == 2 + assert set(df["label"]) == {"positive", "neutral"} + + def test_repaired_truncated_row_gets_repair_note(self, mocker): + """A repaired row should have a repair_note column in the output.""" + line = _make_response_line( + "quote-00001", '{"label":"positive', "Truncated." + ) + self._setup_mock_client( + mocker, "file-567", (line + "\n").encode("utf-8") + ) + mock_save = mocker.patch("batch_processing.batch_method.save_as_csv") + + get_batch_results("batch-005") + + df = mock_save.call_args[0][0] + assert "repair_note" in df.columns + + def test_no_output_file_calls_handle_batch_fail(self, mocker): + """When output_file_id is None, handle_batch_fail should be called.""" + mock_client = MagicMock() + mocker.patch( + "batch_processing.batch_method.get_client", return_value=mock_client + ) + mock_status = MagicMock() + mock_status.output_file_id = None + mock_client.batches.retrieve.return_value = mock_status + + mock_fail = mocker.patch("batch_processing.batch_method.handle_batch_fail") + + get_batch_results("batch-failed") + + mock_fail.assert_called_once_with(mock_client, mock_status) + + def test_authentication_error_propagates(self, mocker): + """If get_client raises (e.g., auth failure), the exception bubbles up.""" + mocker.patch( + "batch_processing.batch_method.get_client", + side_effect=Exception("OpenAI API key not configured"), + ) + with pytest.raises(Exception, match="API key not configured"): + get_batch_results("batch-auth-fail") + + def test_rate_limit_error_propagates(self, mocker): + """A rate-limit exception from the API client bubbles up.""" + mock_client = MagicMock() + mocker.patch( + "batch_processing.batch_method.get_client", return_value=mock_client + ) + mock_status = MagicMock() + mock_status.output_file_id = "file-ratelimit" + mock_client.batches.retrieve.return_value = mock_status + mock_client.files.content.side_effect = Exception("Rate limit exceeded") + + with pytest.raises(Exception, match="Rate limit exceeded"): + get_batch_results("batch-rate-limit") + + def test_connection_error_propagates(self, mocker): + """A network/connection error from the API client bubbles up.""" + mock_client = MagicMock() + mocker.patch( + "batch_processing.batch_method.get_client", return_value=mock_client + ) + mock_status = MagicMock() + mock_status.output_file_id = "file-conn" + mock_client.batches.retrieve.return_value = mock_status + mock_client.files.content.side_effect = ConnectionError("Connection refused") + + with pytest.raises(ConnectionError): + get_batch_results("batch-conn-error") + + def test_unexpected_response_schema_row_excluded(self, mocker): + """A row with an unexpected but valid JSON schema still goes through _safe_parse.""" + # Valid JSON but not the expected schema – save_as_csv still called + line = _make_response_line( + "quote-00001", '{"unexpected_field": "value"}', "Quote." + ) + self._setup_mock_client( + mocker, "file-schema", (line + "\n").encode("utf-8") + ) + mock_save = mocker.patch("batch_processing.batch_method.save_as_csv") + + get_batch_results("batch-schema") + + mock_save.assert_called_once() + df = mock_save.call_args[0][0] + # unexpected_field was parsed and included + assert "unexpected_field" in df.columns + + def test_fenced_json_in_response(self, mocker): + """A response with ```json fences is correctly parsed.""" + line = _make_response_line( + "quote-00001", + "```json\n{\"label\": \"positive\"}\n```", + "Good.", + ) + self._setup_mock_client( + mocker, "file-fenced", (line + "\n").encode("utf-8") + ) + mock_save = mocker.patch("batch_processing.batch_method.save_as_csv") + + get_batch_results("batch-fenced") + + df = mock_save.call_args[0][0] + assert df["label"].iloc[0] == "positive" + + def test_empty_file_content_saves_empty_df(self, mocker): + """An output file with no non-empty lines produces an empty DataFrame.""" + self._setup_mock_client(mocker, "file-empty", b"\n\n") + mock_save = mocker.patch("batch_processing.batch_method.save_as_csv") + + get_batch_results("batch-empty") + + df = mock_save.call_args[0][0] + assert len(df) == 0 diff --git a/tests/test_data_conversion.py b/tests/test_data_conversion.py new file mode 100644 index 0000000..012158b --- /dev/null +++ b/tests/test_data_conversion.py @@ -0,0 +1,179 @@ +""" +Tests for file_handling/data_conversion.py + +Covers: +- make_str_enum: creates valid Enum, handles unicode labels, rejects unknown values +- to_long_df: from dicts, from pydantic models, list-column explode, empty input, + nested dicts, multi-label expansion +- join_datasets: None, string, bytes, iterable, mixed types +""" + +import math +from enum import Enum + +import pandas as pd +import pytest +from pydantic import BaseModel, ConfigDict + +from file_handling.data_conversion import join_datasets, make_str_enum, to_long_df + + +# --------------------------------------------------------------------------- +# make_str_enum +# --------------------------------------------------------------------------- + +class TestMakeStrEnum: + def test_returns_enum_type(self, sample_labels): + EnumCls = make_str_enum("Label", sample_labels) + assert issubclass(EnumCls, Enum) + + def test_members_match_input(self, sample_labels): + EnumCls = make_str_enum("Label", sample_labels) + member_values = [m.value for m in EnumCls] + assert sorted(member_values) == sorted(sample_labels) + + def test_name_is_set(self, sample_labels): + EnumCls = make_str_enum("MySentiment", sample_labels) + assert EnumCls.__name__ == "MySentiment" + + def test_member_access_by_value(self, sample_labels): + EnumCls = make_str_enum("Label", sample_labels) + assert EnumCls("positive").value == "positive" + + def test_single_label(self): + EnumCls = make_str_enum("Label", ["only_label"]) + assert len(list(EnumCls)) == 1 + + def test_unicode_labels(self): + labels = ["positivo", "négative", "中立", "Ünïcödé"] + EnumCls = make_str_enum("Label", labels) + member_values = [m.value for m in EnumCls] + assert "中立" in member_values + assert "Ünïcödé" in member_values + + def test_label_with_spaces(self): + labels = ["very positive", "somewhat negative", "totally neutral"] + EnumCls = make_str_enum("Label", labels) + assert EnumCls("very positive").value == "very positive" + + def test_pydantic_model_enforces_enum(self, sample_labels): + """Enum produced by make_str_enum should integrate with pydantic validation.""" + LabelEnum = make_str_enum("Label", sample_labels) + + class Row(BaseModel): + label: LabelEnum + model_config = ConfigDict(use_enum_values=True) + + row = Row(label="positive") + assert row.label == "positive" + + def test_pydantic_model_rejects_unknown_label(self, sample_labels): + """An unknown label should fail pydantic validation.""" + from pydantic import ValidationError + + LabelEnum = make_str_enum("Label", sample_labels) + + class Row(BaseModel): + label: LabelEnum + model_config = ConfigDict(use_enum_values=True) + + with pytest.raises(ValidationError): + Row(label="completely_unknown") + + +# --------------------------------------------------------------------------- +# to_long_df +# --------------------------------------------------------------------------- + +class TestToLongDf: + def test_from_plain_dicts(self): + records = [{"label": "positive", "quote": "Good!"}, {"label": "negative", "quote": "Bad!"}] + df = to_long_df(records) + assert list(df.columns) == ["label", "quote"] + assert df["label"].tolist() == ["positive", "negative"] + + def test_from_empty_list(self): + df = to_long_df([]) + assert isinstance(df, pd.DataFrame) + assert len(df) == 0 + + def test_from_pydantic_models(self, sample_labels): + LabelEnum = make_str_enum("Label", sample_labels) + + class Row(BaseModel): + id: int + label: LabelEnum + model_config = ConfigDict(use_enum_values=True) + + rows = [Row(id=1, label="positive"), Row(id=2, label="negative")] + df = to_long_df(rows) + assert len(df) == 2 + assert "label" in df.columns + assert df["label"].tolist() == ["positive", "negative"] + + def test_explodes_list_column(self): + """Multi-label records should be exploded into one row per label.""" + records = [ + {"id": 1, "quote": "Good bad neutral", "label": ["positive", "negative"]}, + {"id": 2, "quote": "Just good", "label": ["positive"]}, + ] + df = to_long_df(records) + # Row 1 yields 2 exploded rows, row 2 yields 1 → total 3 + assert len(df) == 3 + assert set(df["label"]) == {"positive", "negative"} + + def test_flattens_nested_dicts(self): + records = [{"meta": {"author": "Alice"}, "label": "positive"}] + df = to_long_df(records) + # json_normalize should create a 'meta.author' column + assert "meta.author" in df.columns + + def test_single_record(self): + df = to_long_df([{"label": "neutral"}]) + assert len(df) == 1 + assert df["label"].iloc[0] == "neutral" + + def test_nonascii_values_preserved(self): + records = [{"label": "中立", "text": "日本語テキスト"}] + df = to_long_df(records) + assert df["label"].iloc[0] == "中立" + assert df["text"].iloc[0] == "日本語テキスト" + + def test_repair_note_column_preserved(self): + records = [{"label": "positive", "quote": "Good!", "repair_note": "Truncated JSON repaired"}] + df = to_long_df(records) + assert "repair_note" in df.columns + assert df["repair_note"].iloc[0] == "Truncated JSON repaired" + + +# --------------------------------------------------------------------------- +# join_datasets +# --------------------------------------------------------------------------- + +class TestJoinDatasets: + def test_none_returns_empty_string(self): + assert join_datasets(None) == "" + + def test_string_returned_as_is(self): + assert join_datasets("my_dataset") == "my_dataset" + + def test_bytes_returned_as_is(self): + # bytes is a Sequence but join_datasets returns it as-is (str/bytes check) + result = join_datasets(b"raw") + assert result == b"raw" + + def test_tuple_joined_with_comma(self): + assert join_datasets(("labels", "quotes")) == "labels,quotes" + + def test_list_joined_with_comma(self): + assert join_datasets(["a", "b", "c"]) == "a,b,c" + + def test_single_item_iterable(self): + assert join_datasets(["only"]) == "only" + + def test_integer_converted_to_string(self): + assert join_datasets(42) == "42" + + def test_generator_joined(self): + result = join_datasets(x for x in ["x", "y"]) + assert result == "x,y" diff --git a/tests/test_data_import.py b/tests/test_data_import.py new file mode 100644 index 0000000..97b64ed --- /dev/null +++ b/tests/test_data_import.py @@ -0,0 +1,177 @@ +""" +Tests for file_handling/data_import.py (non-GUI utilities only) + +These tests exercise the tabular-reading helpers that do not require a +live Tkinter display or user interaction. The GUI import-wizard +(import_data) is excluded because it is driven entirely by user interaction. + +Covers: +- _sniff_delimiter: comma vs. tab vs. semi-colon +- _read_text_table: basic CSV, no-header, max_rows limit, non-ASCII content, + missing columns, empty file +- _load_tabular: dispatch to _read_text_table for .csv/.tsv/.txt and + _read_excel for .xlsx +""" + +import csv +import io +from pathlib import Path + +import pandas as pd +import pytest + +from file_handling.data_import import ( + _load_tabular, + _read_text_table, + _sniff_delimiter, +) + + +# --------------------------------------------------------------------------- +# _sniff_delimiter +# --------------------------------------------------------------------------- + +class TestSniffDelimiter: + def test_detects_comma(self): + sample = "a,b,c\n1,2,3\n" + assert _sniff_delimiter(sample) == "," + + def test_detects_tab(self): + sample = "a\tb\tc\n1\t2\t3\n" + assert _sniff_delimiter(sample) == "\t" + + def test_detects_semicolon(self): + sample = "a;b;c\n1;2;3\n" + assert _sniff_delimiter(sample) == ";" + + def test_fallback_to_comma_on_ambiguous(self): + # Equal counts of commas and tabs → implementation falls back + result = _sniff_delimiter("x") + assert result in (",", "\t") # either is acceptable as a fallback + + +# --------------------------------------------------------------------------- +# _read_text_table +# --------------------------------------------------------------------------- + +class TestReadTextTable: + def test_reads_basic_csv(self, tmp_csv): + path = tmp_csv("col1,col2\nalpha,1\nbeta,2\n") + rows = _read_text_table(str(path)) + assert rows[0] == ["col1", "col2"] + assert rows[1] == ["alpha", "1"] + assert rows[2] == ["beta", "2"] + + def test_reads_basic_tsv(self, tmp_csv): + path = tmp_csv("col1\tcol2\nalpha\t1\nbeta\t2\n", filename="test.tsv") + rows = _read_text_table(str(path)) + assert rows[0] == ["col1", "col2"] + + def test_max_rows_respected(self, tmp_csv): + content = "h1,h2\n" + "\n".join(f"r{i},v{i}" for i in range(50)) + path = tmp_csv(content) + rows = _read_text_table(str(path), max_rows=5) + assert len(rows) == 5 + + def test_nonascii_content_preserved(self, tmp_csv): + path = tmp_csv("text\nCafé\nRésumé\n日本語\n") + rows = _read_text_table(str(path)) + values = [r[0] for r in rows[1:]] + assert "Café" in values + assert "日本語" in values + + def test_single_column_no_header(self, tmp_csv): + path = tmp_csv("positive\nnegative\nneutral\n") + rows = _read_text_table(str(path)) + flat = [r[0] for r in rows] + assert "positive" in flat + assert "negative" in flat + + def test_empty_file_returns_empty_list(self, tmp_csv): + path = tmp_csv("") + rows = _read_text_table(str(path)) + assert rows == [] + + def test_mismatched_row_widths(self, tmp_csv): + """Rows with fewer columns than the header should be read without crashing.""" + path = tmp_csv("a,b,c\n1,2\n3,4,5\n") + rows = _read_text_table(str(path)) + # At least 3 rows (header + 2 data) + assert len(rows) >= 3 + + def test_punctuation_heavy_content(self, tmp_csv): + path = tmp_csv('text\n"He said, \\"It\'s great!\\" — really? (Yes!)"') + rows = _read_text_table(str(path)) + assert len(rows) >= 2 + + def test_long_text_cell(self, tmp_csv): + long_text = "word " * 500 + path = tmp_csv(f"text\n{long_text}\n") + rows = _read_text_table(str(path)) + assert len(rows) == 2 + assert rows[1][0].strip() == long_text.strip() + + +# --------------------------------------------------------------------------- +# _load_tabular +# --------------------------------------------------------------------------- + +class TestLoadTabular: + def test_csv_dispatches_correctly(self, tmp_csv): + path = tmp_csv("name,value\nfoo,1\nbar,2\n") + rows = _load_tabular(str(path)) + assert rows[0] == ["name", "value"] + assert len(rows) == 3 + + def test_tsv_dispatches_correctly(self, tmp_csv): + path = tmp_csv("name\tvalue\nfoo\t1\n", filename="data.tsv") + rows = _load_tabular(str(path)) + assert rows[0] == ["name", "value"] + + def test_max_rows_forwarded(self, tmp_csv): + content = "h\n" + "\n".join(str(i) for i in range(100)) + path = tmp_csv(content) + rows = _load_tabular(str(path), max_rows=10) + assert len(rows) == 10 + + def test_excel_dispatches_correctly(self, tmp_path): + """_load_tabular should read .xlsx files via pandas/openpyxl. + + Note: _read_excel uses pd.read_excel which consumes the header row + internally, so itertuples yields only data rows (no header row in + the returned list). + """ + xlsx_path = tmp_path / "test.xlsx" + df = pd.DataFrame({"col_a": ["x", "y"], "col_b": [1, 2]}) + df.to_excel(xlsx_path, index=False) + + rows = _load_tabular(str(xlsx_path)) + # pandas consumed the header; the first element is the first data row + assert rows[0] == ["x", "1"] + assert rows[1] == ["y", "2"] + + def test_unsupported_extension_raises(self, tmp_path): + """An unsupported extension whose content also fails pandas CSV + parsing should raise RuntimeError.""" + weird = tmp_path / "file.unknown_ext_xyz" + # Write binary content that pandas cannot parse as CSV + weird.write_bytes(b"\x00\x01\x02\x03\xff\xfe") + with pytest.raises(RuntimeError): + _load_tabular(str(weird)) + + def test_realistic_fixture_csv(self): + """Use the checked-in realistic fixture to verify end-to-end reading.""" + fixture = Path(__file__).parent / "fixtures" / "realistic_dataset.csv" + rows = _load_tabular(str(fixture)) + # Header row + assert rows[0][0] == "text" + # At least 5 data rows + assert len(rows) >= 6 + + def test_label_fixture_single_column(self): + fixture = Path(__file__).parent / "fixtures" / "sample_labels.csv" + rows = _load_tabular(str(fixture)) + values = [r[0] for r in rows[1:]] # skip header + assert "positive" in values + assert "negative" in values + assert "neutral" in values diff --git a/tests/test_reliability.py b/tests/test_reliability.py new file mode 100644 index 0000000..179d182 --- /dev/null +++ b/tests/test_reliability.py @@ -0,0 +1,86 @@ +""" +Tests for live_processing/reliability_calculator.py (pure-logic functions only) + +Covers: +- compute_cohens_kappa: perfect agreement, zero agreement, known value, + mismatched series lengths, single label, empty series +""" + +import math + +import pandas as pd +import pytest + +from live_processing.reliability_calculator import compute_cohens_kappa + + +class TestComputeCohensKappa: + def test_perfect_agreement_returns_one(self): + s1 = pd.Series(["pos", "neg", "neu", "pos"]) + s2 = pd.Series(["pos", "neg", "neu", "pos"]) + kappa = compute_cohens_kappa(s1, s2) + assert kappa == pytest.approx(1.0) + + def test_empty_series_returns_nan(self): + kappa = compute_cohens_kappa(pd.Series([], dtype=str), pd.Series([], dtype=str)) + assert math.isnan(kappa) + + def test_mismatched_lengths_returns_nan(self): + s1 = pd.Series(["pos", "neg"]) + s2 = pd.Series(["pos"]) + kappa = compute_cohens_kappa(s1, s2) + assert math.isnan(kappa) + + def test_single_unique_label_perfect_agreement(self): + """All items the same label → kappa is 1 (pe would be 1, handled by guard).""" + s1 = pd.Series(["pos"] * 10) + s2 = pd.Series(["pos"] * 10) + kappa = compute_cohens_kappa(s1, s2) + # When pe == 1.0, the implementation returns 1.0 by convention + assert kappa == pytest.approx(1.0) + + def test_known_kappa_value(self): + """ + 2-class problem: 60 items, balanced coders. + + Confusion matrix: + coder_b_pos coder_b_neg + coder_a_pos 20 10 + coder_a_neg 10 20 + + n = 60 + po = 40/60 + row_marg = [30/60, 30/60], col_marg = [30/60, 30/60] + pe = (30/60)^2 + (30/60)^2 = 0.5 + kappa = (2/3 - 0.5) / (1 - 0.5) = (1/6) / (1/2) = 1/3 + """ + s1 = pd.Series(["pos"] * 30 + ["neg"] * 30) + s2 = pd.Series(["pos"] * 20 + ["neg"] * 10 + ["pos"] * 10 + ["neg"] * 20) + kappa = compute_cohens_kappa(s1, s2) + assert kappa == pytest.approx(1 / 3, abs=1e-6) + + def test_result_between_negative_one_and_one(self): + """kappa should always be in [-1, 1].""" + s1 = pd.Series(["a", "b", "c", "a", "b"]) + s2 = pd.Series(["b", "c", "a", "b", "a"]) + kappa = compute_cohens_kappa(s1, s2) + assert -1.0 <= kappa <= 1.0 + + def test_partial_agreement_is_between_zero_and_one(self): + s1 = pd.Series(["pos", "pos", "neg", "neg", "neu", "neu"]) + s2 = pd.Series(["pos", "neg", "neg", "neg", "neu", "pos"]) + kappa = compute_cohens_kappa(s1, s2) + assert 0.0 <= kappa <= 1.0 + + def test_asymmetric_label_sets(self): + """Coders using different subsets of labels should not crash.""" + s1 = pd.Series(["pos", "pos", "neg"]) + s2 = pd.Series(["pos", "neu", "neg"]) + kappa = compute_cohens_kappa(s1, s2) + assert not math.isnan(kappa) + + def test_unicode_labels(self): + s1 = pd.Series(["中立", "positivo", "négative"]) + s2 = pd.Series(["中立", "positivo", "négative"]) + kappa = compute_cohens_kappa(s1, s2) + assert kappa == pytest.approx(1.0) diff --git a/tests/test_settings.py b/tests/test_settings.py new file mode 100644 index 0000000..3666dc1 --- /dev/null +++ b/tests/test_settings.py @@ -0,0 +1,172 @@ +""" +Tests for settings/user_config.py + +Covers: +- load_user_settings: returns {} when no file exists, returns data when it does +- save_user_settings: persists data to disk, overwrites on second save +- get_setting: prefers user setting over config default, falls back to + config attribute, falls back to provided default +- set_setting: persists individual key, does not clobber other keys +- get_user_config_file: returns a path inside get_user_config_dir() +""" + +import json +from pathlib import Path + +import pytest + +from settings.user_config import ( + get_setting, + get_user_config_dir, + get_user_config_file, + load_user_settings, + save_user_settings, + set_setting, +) + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + +@pytest.fixture(autouse=True) +def isolated_config_dir(tmp_path, monkeypatch): + """Redirect all user-config reads/writes to a temp directory.""" + config_dir = tmp_path / "CodebookAI" + config_dir.mkdir() + monkeypatch.setattr( + "settings.user_config.get_user_config_dir", lambda: config_dir + ) + monkeypatch.setattr( + "settings.user_config.get_user_config_file", + lambda: config_dir / "config.json", + ) + return config_dir + + +# --------------------------------------------------------------------------- +# load_user_settings +# --------------------------------------------------------------------------- + +class TestLoadUserSettings: + def test_returns_empty_dict_when_no_file(self): + assert load_user_settings() == {} + + def test_returns_saved_data(self, isolated_config_dir): + cfg = isolated_config_dir / "config.json" + cfg.write_text(json.dumps({"model": "gpt-5", "max_batches": 10})) + result = load_user_settings() + assert result["model"] == "gpt-5" + assert result["max_batches"] == 10 + + def test_returns_empty_dict_on_invalid_json(self, isolated_config_dir): + cfg = isolated_config_dir / "config.json" + cfg.write_text("NOT VALID JSON {{{") + result = load_user_settings() + assert result == {} + + def test_returns_empty_dict_on_empty_file(self, isolated_config_dir): + cfg = isolated_config_dir / "config.json" + cfg.write_text("") + result = load_user_settings() + assert result == {} + + +# --------------------------------------------------------------------------- +# save_user_settings +# --------------------------------------------------------------------------- + +class TestSaveUserSettings: + def test_roundtrip(self): + data = {"model": "gpt-4o-mini", "max_batches": 4, "time_zone": "UTC"} + save_user_settings(data) + assert load_user_settings() == data + + def test_overwrite_on_second_save(self): + save_user_settings({"key": "first"}) + save_user_settings({"key": "second"}) + assert load_user_settings()["key"] == "second" + + def test_unicode_values_preserved(self): + data = {"label": "Ünïcödé — 日本語"} + save_user_settings(data) + assert load_user_settings()["label"] == "Ünïcödé — 日本語" + + def test_nested_dict_preserved(self): + data = {"prefs": {"theme": "dark", "font_size": 14}} + save_user_settings(data) + loaded = load_user_settings() + assert loaded["prefs"]["theme"] == "dark" + + def test_empty_dict_saved(self): + save_user_settings({}) + assert load_user_settings() == {} + + +# --------------------------------------------------------------------------- +# get_setting +# --------------------------------------------------------------------------- + +class TestGetSetting: + def test_returns_default_config_attribute(self): + # 'model' is defined in settings/config.py as 'gpt-4o-mini' + result = get_setting("model") + assert result == "gpt-4o-mini" + + def test_user_setting_overrides_config(self): + save_user_settings({"model": "gpt-5-override"}) + assert get_setting("model") == "gpt-5-override" + + def test_returns_provided_default_when_key_missing(self): + result = get_setting("nonexistent_key_xyz", default="my_default") + assert result == "my_default" + + def test_returns_none_when_key_missing_and_no_default(self): + result = get_setting("nonexistent_key_xyz") + assert result is None + + def test_user_setting_not_in_config_returned(self): + save_user_settings({"custom_key": "custom_value"}) + assert get_setting("custom_key") == "custom_value" + + +# --------------------------------------------------------------------------- +# set_setting +# --------------------------------------------------------------------------- + +class TestSetSetting: + def test_sets_single_key(self): + set_setting("my_key", "my_value") + assert get_setting("my_key") == "my_value" + + def test_does_not_clobber_other_keys(self): + save_user_settings({"key_a": "a", "key_b": "b"}) + set_setting("key_a", "updated") + assert get_setting("key_b") == "b" + + def test_overwrites_existing_key(self): + set_setting("x", "first") + set_setting("x", "second") + assert get_setting("x") == "second" + + def test_sets_integer_value(self): + set_setting("max_batches", 99) + assert get_setting("max_batches") == 99 + + def test_sets_boolean_value(self): + set_setting("feature_flag", True) + assert get_setting("feature_flag") is True + + +# --------------------------------------------------------------------------- +# Path helpers +# --------------------------------------------------------------------------- + +class TestPathHelpers: + def test_config_file_inside_config_dir(self): + # Access functions through the module so any monkeypatching applied by + # the autouse isolated_config_dir fixture is respected for both calls. + import settings.user_config as uc + cfg_dir = uc.get_user_config_dir() + cfg_file = uc.get_user_config_file() + assert str(cfg_file).startswith(str(cfg_dir)) From 07d13bceca09412602582d3437a8a37c38471ee7 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 30 Apr 2026 16:49:46 +0000 Subject: [PATCH 2/2] fix: add explicit permissions: contents: read to CI workflow Agent-Logs-Url: https://github.com/tmaier-kettering/CodebookAI/sessions/e942e5e4-04d0-4469-bc8e-c6be9e3afdaa Co-authored-by: tmaier-kettering <109093855+tmaier-kettering@users.noreply.github.com> --- .github/workflows/tests.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 6f2e4cb..49d6688 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -6,6 +6,9 @@ on: pull_request: branches: ["**"] +permissions: + contents: read + jobs: test: name: Test (Python ${{ matrix.python-version }})