diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..6fca06b --- /dev/null +++ b/.gitattributes @@ -0,0 +1,3 @@ +# Shell scripts must keep LF endings to run on macOS/Linux/CI. +*.sh text eol=lf +Makefile text eol=lf diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..c5253fd --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,49 @@ +name: CI + +on: + pull_request: + branches: [main] + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: ci-${{ github.ref }} + cancel-in-progress: true + +jobs: + backend: + name: Backend (pytest) + runs-on: ubuntu-latest + defaults: + run: + working-directory: backend + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.11" + cache: pip + cache-dependency-path: | + backend/requirements.txt + backend/requirements-dev.txt + - run: pip install -r requirements-dev.txt + - run: python -m pytest + + frontend: + name: Frontend (lint + build) + runs-on: ubuntu-latest + defaults: + run: + working-directory: frontend + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: "20" + cache: npm + cache-dependency-path: frontend/package-lock.json + - run: npm ci + - run: npm run lint + - run: npm run build diff --git a/.gitignore b/.gitignore index e985853..6f6ddea 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,33 @@ .vercel + +# Secrets / local config (commit only *.env.example) +.env +.env.* +!.env.example +!frontend/.env.production + +# Python +__pycache__/ +*.py[cod] +.venv/ +venv/ +.pytest_cache/ +*.egg-info/ + +# Backend runtime state (vector store, SQLite) +backend/app/chroma_db/ +backend/*.db + +# Scraper output (committed corpus lives in data/) +backend/scrapers/catalog/output/ +backend/scrapers/catalog/test_output/ + +# Node +node_modules/ + +# IDE / OS +.vscode/ +.idea/ +.DS_Store +Thumbs.db +*.log diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..5f07b18 --- /dev/null +++ b/Makefile @@ -0,0 +1,4 @@ +# Thin wrapper; the logic lives in scripts/dev.sh (Windows: scripts/dev.ps1). +.PHONY: setup seed api web test +setup seed api web test: + @scripts/dev.sh $@ diff --git a/README.md b/README.md new file mode 100644 index 0000000..113b7ed --- /dev/null +++ b/README.md @@ -0,0 +1,145 @@ +# Ask Temoc + +Ask Temoc is a chatbot that answers UT Dallas students' questions about degree +programs, grounded in the UTD undergraduate catalog. It's built by the UTD AI +Society. + +## Architecture + +``` +browser ──► Next.js (frontend/) ──► FastAPI (backend/) ──► Chroma vector store + /api/proxy/* rewrite RAG chain (LangChain) seeded from data/catalog + OpenAI or Ollama +``` + +- **The browser calls the web app's own `/api/proxy/*`.** Next.js forwards those + requests to the API (`ASKTEMOC_API_ORIGIN`): `http://localhost:8000` under + `npm run dev`, Cloud Run in production builds. +- **The API** retrieves catalog chunks from Chroma and streams an answer with its + sources as server-sent events. +- **Models.** If `OPENAI_API_KEY` is set, the API uses OpenAI for chat and + embeddings; otherwise it uses a local [Ollama](https://ollama.com). +- **No sign-in yet.** There's no app-level sign-in; it's planned for Weeks 2-4 of + the 10-week plan. See [docs/KNOWN_ISSUES.md](docs/KNOWN_ISSUES.md). + +``` +frontend/ Next.js 15 chat UI +backend/ FastAPI app (app/), seed script (scripts/), tests, catalog scraper (scrapers/) +data/ committed scraped corpus: data/catalog//{requirements,example}.txt +docs/ KNOWN_ISSUES.md +scripts/ dev.sh / dev.ps1 task runners +``` + +## Quickstart + +You need **Python 3.11**, **Node 20+**, and Docker (optional). You also need +**either** an OpenAI API key with billing enabled **or** a local Ollama. + +All commands below run from the repo root. They're written for macOS/Linux/Git +Bash; on Windows PowerShell, replace `scripts/dev.sh` with `.\scripts\dev.ps1`. + +**1. Clone and install.** + +```bash +git clone https://github.com/aisutd/asktemoc.git +cd asktemoc +scripts/dev.sh setup +``` + +This creates `backend/.venv` and installs backend and frontend dependencies. It +also copies `backend/.env.example` to `backend/.env` and `frontend/.env.example` +to `frontend/.env.local`. Both copies are gitignored; never commit them. + +**2. Choose a model provider** in `backend/.env`. Either: +- set `OPENAI_API_KEY`, or +- leave it empty and pull the two default Ollama models: + `ollama pull llama3.1:8b && ollama pull nomic-embed-text`. + +Every other setting has a working default. The frontend needs no env vars for +local development. + +**3. Seed the vector store** from the catalog files in `data/catalog/`: + +```bash +scripts/dev.sh seed +``` + +This prints document and chunk counts (193 documents and 2047 chunks today). +Re-running is safe: it replaces the chunks rather than duplicating them. If you +switch between OpenAI and Ollama (or change the embedding model), run this +again; vectors from different models can't be mixed, and the API logs a warning +if they don't match. + +**4. Start the API and the web app** in two terminals: + +```bash +scripts/dev.sh api # http://localhost:8000 +scripts/dev.sh web # http://localhost:3000 +``` + +**5. Try it.** Open , click **Get Started**, and ask +*"What are the core requirements for the BS in Computer Science?"* The answer +streams in with linked sources. + +You can also call the API directly, for example from scripts: + +```bash +curl -N -X POST localhost:8000/api/chat -H 'content-type: application/json' \ + -d '{"message":"core requirements for BS Computer Science"}' +``` + +### Running the API in Docker instead + +```bash +docker compose up --build api +docker compose run --rm api python scripts/seed_catalog.py +``` + +The container reads `backend/.env`. Its vector store and database live in a +Docker volume. It reaches Ollama on your host by default; `docker-compose.yml` +shows how to use an Ollama container instead. + +## Tests and CI + +```bash +scripts/dev.sh test # backend pytest, then frontend lint + build +``` + +The backend tests run offline: the RAG chain is faked, and nothing calls OpenAI, +Ollama or the network. On every pull request to `main`, GitHub Actions runs the +same checks (`.github/workflows/ci.yml`). + +## Chat API + +`POST /api/chat` with body `{"message": "..."}` returns `text/event-stream`. +Each event is one line, `data: {json}`: + +| `type` | Fields | Meaning | +| --- | --- | --- | +| `source` | `message` (`"Source n: "`), `index`, `url`, `title?` | one retrieved chunk, sent before the answer | +| `text` | `message` | the next piece of the answer; append it | +| `done` | | end of stream | + +Clients should ignore unknown types. `GET /api/health` works with or without the +trailing slash. + +## Deployment + +The frontend deploys to Vercel and the API to Cloud Run. + +**Vercel:** `ASKTEMOC_API_ORIGIN` is already committed in `vercel.json` and +`frontend/.env.production`, so no extra variables are needed. Until app-level +sign-in exists, visitors are gated by Vercel Deployment Protection (Project → +Deployment Protection → Vercel Authentication). Run `vercel link` / +`vercel deploy` from the repository root with the project's Root Directory set +to `frontend`. + +**Cloud Run:** +- `ENVIRONMENT=production` +- `OPENAI_API_KEY` +- `CORS_ORIGINS=https://` +- `DATABASE_URL`, plus `CHROMA_PERSIST_DIRECTORY` on persistent storage, seeded + with `scripts/seed_catalog.py` + +The API has no authentication, so anyone who knows its URL can use it. See +[docs/KNOWN_ISSUES.md](docs/KNOWN_ISSUES.md). diff --git a/backend/.dockerignore b/backend/.dockerignore new file mode 100644 index 0000000..525d5b6 --- /dev/null +++ b/backend/.dockerignore @@ -0,0 +1,11 @@ +.env +**/.env +**/__pycache__ +**/*.pyc +.venv +venv +app/chroma_db +*.db +scrapers +tests +.pytest_cache diff --git a/backend/.env.example b/backend/.env.example new file mode 100644 index 0000000..5a1fed0 --- /dev/null +++ b/backend/.env.example @@ -0,0 +1,29 @@ +# Copy to backend/.env and fill in. backend/.env is gitignored; never commit it. +# Every value here is optional unless marked required; defaults are shown. + +ENVIRONMENT=development + +# --- LLM + embeddings ------------------------------------------------------ +# If OPENAI_API_KEY is set, chat and embeddings use OpenAI; otherwise Ollama. +# Switching provider (or embedding model) requires re-running +# `python scripts/seed_catalog.py`: vectors from different models don't mix. +OPENAI_API_KEY= +OPENAI_MODEL=gpt-4o-mini +OPENAI_EMBEDDING_MODEL=text-embedding-3-small + +OLLAMA_BASE_URL=http://localhost:11434 +OLLAMA_MODEL=llama3.1:8b +OLLAMA_EMBEDDING_MODEL=nomic-embed-text + +# --- Storage ---------------------------------------------------------------- +# Relative paths resolve against backend/. Postgres works by changing the URL, +# e.g. postgresql+psycopg://user:pass@host/asktemoc (install the driver). +DATABASE_URL=sqlite:///./asktemoc.db +DB_ECHO=false +CHROMA_PERSIST_DIRECTORY=./app/chroma_db +CHROMA_COLLECTION_NAME=asktemoc_collection + +# --- HTTP ------------------------------------------------------------------- +# Comma-separated. The browser only talks to the Next.js server, so this rarely +# needs more than the web origin. +CORS_ORIGINS=http://localhost:3000 diff --git a/backend/.gitignore b/backend/.gitignore deleted file mode 100644 index 7842a1b..0000000 --- a/backend/.gitignore +++ /dev/null @@ -1,25 +0,0 @@ -# Python -__pycache__/ -*.py[cod] -*$py.class -*.so -.Python -env/ -venv/ -ENV/ -venv - -# Output -output/ -*.log - -# IDE -.vscode/ -.idea/ -*.swp -*.swo - -# OS -.DS_Store -Thumbs.db - diff --git a/backend/CHANGES.md b/backend/CHANGES.md new file mode 100644 index 0000000..767a813 --- /dev/null +++ b/backend/CHANGES.md @@ -0,0 +1,388 @@ +# Changelog + +## Date: February 17, 2026 + +- **feat**: Migrated from Pinecone to ChromaDB for vector storage, removing external API dependency and simplifying the architecture + +### Files Created + +1. **app/services/chroma_service.py** (NEW) - 264 lines + - ChromaService class replacing PineconeExportService + - Local persistent storage using ChromaDB + - Methods: upsert_vectors, export_document_embeddings, export_unsynced_embeddings, delete_from_chroma, search_chroma, get_collection_stats + +2. **app/api/endpoints/chroma.py** (NEW) - 175 lines + - 6 ChromaDB API endpoints replacing Pinecone endpoints + - Endpoints: export/document, export/unsynced, export/batch, delete vectors, index stats, search + +### Files Modified + +1. **app/db/models.py** (MODIFIED) + - Renamed `pinecone_id` column to `chroma_id` in Embedding model + - Updated index from `idx_pinecone_id` to `idx_chroma_id` + - Updated repr method + +2. **app/db/services.py** (MODIFIED) + - Updated EmbeddingService methods to use `chroma_id` instead of `pinecone_id` + - Methods updated: create_embedding, update_embedding, mark_synced, list_unsynced_embeddings + +3. **app/services/document_management.py** (MODIFIED) + - Updated export_document_to_json to use `chroma_id` + - Updated get_sync_status_summary docstring to reference ChromaDB + +4. **app/schemas/db_schemas.py** (MODIFIED) + - Renamed PineconeExportResponse → ChromaExportResponse + - Renamed PineconeIndexStats → ChromaIndexStats + - Updated EmbeddingUpdate and EmbeddingResponse to use `chroma_id` + - Updated BatchEmbeddingSync docstring + +5. **app/api/endpoints/documents.py** (MODIFIED) + - Updated update_embedding endpoint to pass `chroma_id` + +6. **app/api/endpoints/dashboard.py** (MODIFIED) + - Updated sync-status endpoint docstring + +7. **app/main.py** (MODIFIED) + - Changed import from pinecone to chroma + - Updated router inclusion + +8. **.env.example** (MODIFIED) + - Removed Pinecone configuration variables + - Kept ChromaDB configuration + +9. **app/.env** (MODIFIED) + - Removed Pinecone configuration variables + +### Files Deleted + +1. **app/services/pinecone_service.py** (DELETED) + - Removed PineconeExportService class + +2. **app/api/endpoints/pinecone.py** (DELETED) + - Removed Pinecone API endpoints + +### API Changes + +- Endpoint prefix changed from `/api/pinecone` to `/api/chroma` +- All endpoint functionality preserved with ChromaDB backend + +### Configuration Changes + +- Removed: PINECONE_API_KEY, PINECONE_ENVIRONMENT, PINECONE_INDEX_NAME +- Using: CHROMA_PERSIST_DIRECTORY, CHROMA_COLLECTION_NAME + +### Benefits + +- No external API dependency +- Local persistent storage +- Simplified architecture +- Cost reduction (no Pinecone subscription needed) + +## Date: November 20, 2025 + +- **feat**: Connected ChromaDB RAG pipeline to existing asktemoc.db database with university program data +- **feat**: Integrated Ollama LLM with streaming API for real-time responses +- **fix**: Resolved import path issues from relative to absolute imports + +### Files Modified/Created: + +#### Data Ingestion Service +1. **app/services/data_ingestion_service.py** (CREATED) + - Implemented data ingestion service for ChromaDB + - Successfully ingested 116 university program files (1483 document chunks) + - Created proper document chunking and embedding generation + +#### Environment Configuration +2. **app/.env** (CREATED) + - Added ChromaDB persistence configuration + - Configured Ollama model settings (OLLAMA_MODEL, OLLAMA_EMBEDDING_MODEL) + +#### RAG Pipeline Updates +3. **app/services/retriever_service.py** (MODIFIED) + - Updated to use chroma_db/chroma.sqlite3 persistence path + - Enhanced ChromaDB integration with Ollama embeddings +4. **app/services/rag_chain_service.py** (MODIFIED) + - Fixed import paths from relative to absolute imports + - Enhanced RAG chain with proper ChromaDB retriever integration +5. **app/services/prompt_service.py** (WORKING) + - Existing prompt template working correctly + +#### API Endpoint Updates +6. **app/api/endpoints/rag_endpoint.py** (MODIFIED) + - Enhanced to work with real RAG pipeline instead of mock data + - Streams responses from actual Ollama LLM with ChromaDB context + +#### Documentation +7. **plan.md** (MODIFIED) + - Wiped old plan and documented completed implementation status + +## Date: November 19, 2025 + +- **feat**: Implemented LangChain RAG pipeline for conversational AI. + +### Files Modified/Created: + +#### Prompt Template +1. **app/services/prompt_service.py** (MODIFIED) + - Defined a structured prompt template for RAG chain using `langchain_core.prompts.PromptTemplate`. + +#### Services +2. **app/services/retriever_service.py** (CREATED) + - Implemented `RetrieverService` to integrate ChromaDB with `OllamaEmbeddings`. +3. **app/services/rag_chain_service.py** (MODIFIED) + - Constructed the core RAG chain using `Ollama` for LLM and configured to output generated text and source documents. + +#### API Endpoints +4. **app/api/endpoints/rag_endpoint.py** (CREATED) + - Exposed the RAG chain through a new streaming API endpoint `/api/chat`. + - Designed to accept user messages and stream responses, including text and source documents, conforming to frontend requirements. + +#### Core Application +5. **app/main.py** (MODIFIED) + - Integrated the new `rag_endpoint` router. + +#### Testing +6. **tests/test_rag_pipeline.py** (CREATED) + - Added comprehensive unit and integration tests for the RAG pipeline. + - Implemented mocking for `rag_chain_service.get_chain` and `Ollama` components to ensure robust and independent testing. + +--- + +### Summary of Changes + +- **RAG Integration Complete**: Successfully connected ChromaDB with existing university program data (116 files, 1483 chunks) +- **Ollama Integration**: Configured and tested with llama3.1:8b model for text generation +- **Real-time Streaming**: Enhanced `/api/chat` endpoint to stream responses from actual RAG pipeline +- **Import Fixes**: Resolved module resolution issues by converting relative to absolute imports +- **Data Ingestion**: Created service to ingest and embed all university program requirements + +### Dependencies Added +- `langchain-chroma` +- `chromadb` +- `pytest-mock` (for testing) +- `langchain-community` (for Ollama integrations) + +--- + +## Date: November 16, 2025 + +- **a7e7157**: Updated README with setup instructions and included new dependencies to requirements.txt +- **0e514cb**: Added the html document chunking class +- **ab7c5f7**: Added the general web scraping + +--- + +# Implementation Changes Log + +## Date: November 13, 2025 + +### Files Created (9 NEW files) + +#### Database Layer +1. **app/db/__init__.py** (NEW) + - Module initialization with exports + - Clean import interface for database components + +2. **app/db/models.py** (NEW) - 96 lines + - Document ORM model (soft-delete, metadata) + - Chunk ORM model (sequence ordering, metadata) + - Embedding ORM model (vector storage, sync tracking) + - Relationships and cascade delete configuration + +3. **app/db/database.py** (NEW) - 42 lines + - SQLAlchemy engine initialization + - Session factory and dependency injection + - Database initialization and cleanup functions + +4. **app/db/services.py** (NEW) - 328 lines + - DocumentService: 8 CRUD + search methods + - ChunkService: 7 CRUD + batch methods + - EmbeddingService: 10 CRUD + sync methods + +#### Services +5. **app/services/pinecone_service.py** (NEW) - 216 lines + - PineconeExportService class + - Vector preparation with metadata + - Batch upsert to Pinecone + - Vector deletion and search + - Index statistics + +6. **app/services/document_management.py** (NEW) - 305 lines + - DocumentManagementUtils class + - Dashboard and analytics functions + - Document statistics and overview + - Batch operations (delete, duplicate) + - JSON export and content search + - Activity tracking + +#### API Endpoints +7. **app/api/endpoints/documents.py** (NEW) - 287 lines + - Document CRUD endpoints (6) + - Chunk CRUD and batch endpoints (7) + - Embedding CRUD endpoints (4) + - Total: 17 endpoints + +8. **app/api/endpoints/pinecone.py** (NEW) - 172 lines + - Document export endpoint + - Unsynced export endpoint + - Batch export endpoint + - Vector deletion endpoint + - Index statistics endpoint + - Vector search endpoint + - Total: 6 endpoints + +9. **app/api/endpoints/dashboard.py** (NEW) - 99 lines + - Dashboard overview endpoint + - Document statistics endpoint + - Export/duplicate endpoints + - Batch delete endpoint + - Content search endpoint + - Activity tracking endpoint + - Sync status endpoint + - Total: 8 endpoints + +### Files Modified (3 files) + +#### Configuration +1. **requirements.txt** (MODIFIED) + - Added: `pinecone-client==5.0.1` + - No other changes needed + +2. **app/main.py** (MODIFIED) + - Added startup event for database initialization + - Integrated all new routers (documents, pinecone, dashboard) + - Organized by feature + +#### Schemas +3. **app/schemas/db_schemas.py** (MODIFIED) + - Added: DocumentCreate, DocumentUpdate, DocumentResponse, DocumentDetailResponse + - Added: ChunkCreate, ChunkUpdate, ChunkResponse, ChunkDetailResponse + - Added: EmbeddingCreate, EmbeddingUpdate, EmbeddingResponse + - Added: BatchChunkCreate, BatchEmbeddingSync + - Added: PineconeExportResponse, PineconeIndexStats + - Added: DocumentSearch, SearchResponse + - Total: 20+ new Pydantic models + +### Files Created for Documentation (3 files) + +1. **.env.example** (NEW) + - Environment variable template + - Database configuration options + - Pinecone credentials setup + +2. **DATABASE_API_DOCS.md** (NEW) + - 400+ lines of comprehensive documentation + - Database schema design and explanation + - Complete API endpoint reference + - Usage examples (Python and curl) + - Service layer documentation + - Performance considerations + - Troubleshooting guide + +3. **QUICK_START.md** (NEW) + - Quick reference guide + - File structure overview + - Key components explanation + - Common workflows + - Database schema diagrams + - Service class references + +### Files Updated (1 file) + +1. **IMPLEMENTATION_STATUS.md** (NEW) + - Implementation status report + - Comprehensive checklist + - Statistics and metrics + - Verification results + +--- + +## Summary of Changes + +### Code Statistics +- **Total Lines of Code Added**: 1,825 +- **Total Files Created**: 9 (code) + 3 (documentation) +- **Total Files Modified**: 3 + +### Feature Summary + +#### Database (3 tables) +- ✅ Documents table with metadata +- ✅ Chunks table with sequence ordering +- ✅ Embeddings table with Pinecone sync +- ✅ Proper relationships and cascade deletes +- ✅ Soft delete support +- ✅ 6 optimized indexes + +#### Services (40+ methods) +- ✅ DocumentService (8 methods) +- ✅ ChunkService (7 methods) +- ✅ EmbeddingService (10 methods) +- ✅ PineconeExportService (7 methods) +- ✅ DocumentManagementUtils (8 methods) + +#### API Endpoints (40+) +- ✅ Documents endpoints (6) +- ✅ Chunks endpoints (7) +- ✅ Embeddings endpoints (4) +- ✅ Pinecone endpoints (6) +- ✅ Dashboard endpoints (8) + +#### Pinecone Integration +- ✅ Vector upsert with metadata +- ✅ Batch sync operations +- ✅ Sync status tracking +- ✅ Vector deletion +- ✅ Index statistics +- ✅ Vector search + +#### Dashboard Features +- ✅ Document statistics +- ✅ Sync status monitoring +- ✅ Batch operations +- ✅ Content search +- ✅ Activity tracking +- ✅ JSON export + +### Breaking Changes +None - All new functionality, no existing code modified except main.py and requirements.txt + +### Dependencies Added +- `pinecone-client==5.0.1` (for Pinecone integration) + +### Database Migration +None required - Tables created automatically on startup + +--- + +## Verification Checklist + +- ✅ All Python files compile without errors +- ✅ All imports work correctly +- ✅ Database models properly defined +- ✅ Service classes instantiate correctly +- ✅ FastAPI app initializes with all routes +- ✅ Pydantic schemas validate correctly +- ✅ No reserved name conflicts +- ✅ Documentation is comprehensive +- ✅ Code is well-commented + +--- + +## How to Use + +1. Install dependencies: `pip install -r requirements.txt` +2. Set up environment: `cp .env.example .env` +3. Configure Pinecone: Edit `.env` with API key +4. Run server: `uvicorn app.main:app --reload` +5. Access API: `http://localhost:8000` +6. View documentation: `http://localhost:8000/docs` + +--- + +## Documentation + +- **DATABASE_API_DOCS.md** - Complete API reference +- **QUICK_START.md** - Setup and quick reference +- **IMPLEMENTATION_STATUS.md** - Detailed report +- **Inline code comments** - Well-documented methods diff --git a/backend/DATABASE_API_DOCS.md b/backend/DATABASE_API_DOCS.md new file mode 100644 index 0000000..830ea0f --- /dev/null +++ b/backend/DATABASE_API_DOCS.md @@ -0,0 +1,575 @@ +# AskTemoc Backend - Database & API Documentation + +## Overview + +This implementation provides a complete SQLite-based relational schema with FastAPI endpoints for managing documents, chunks, embeddings, and Pinecone export operations. The system is designed for RAG (Retrieval-Augmented Generation) applications with scalable metadata management. + +## Architecture + +### Database Schema + +#### Documents Table (`documents`) +Stores document metadata with soft-delete support. + +| Column | Type | Purpose | +|--------|------|---------| +| id | String (Primary Key) | Unique document identifier (UUID) | +| title | String | Document title (indexed) | +| source | String | Document source (URL, file path, etc.) | +| metadata | JSON | Flexible metadata storage | +| created_at | DateTime | Creation timestamp (indexed) | +| updated_at | DateTime | Last update timestamp | +| is_deleted | Boolean | Soft delete flag (indexed) | + +**Relationships:** +- One-to-Many with `Chunk` (cascade delete) + +#### Chunks Table (`chunks`) +Stores text chunks extracted from documents with sequence ordering. + +| Column | Type | Purpose | +|--------|------|---------| +| id | String (Primary Key) | Unique chunk identifier (UUID) | +| document_id | String (Foreign Key) | Reference to parent document | +| chunk_index | Integer | Sequence position within document (indexed with doc_id) | +| text | Text | Chunk content | +| metadata | JSON | Chunk-specific metadata | +| created_at | DateTime | Creation timestamp (indexed) | +| updated_at | DateTime | Last update timestamp | +| is_deleted | Boolean | Soft delete flag (indexed) | + +**Relationships:** +- Many-to-One with `Document` +- One-to-Many with `Embedding` (cascade delete) + +**Indexes:** +- Composite: (document_id, chunk_index) for efficient chunk retrieval + +#### Embeddings Table (`embeddings`) +Stores embedding vectors and sync status with Pinecone. + +| Column | Type | Purpose | +|--------|------|---------| +| id | String (Primary Key) | Unique embedding identifier (UUID) | +| chunk_id | String (Foreign Key) | Reference to parent chunk | +| vector | JSON | Embedding vector as JSON array | +| model | String | Embedding model name (e.g., "text-embedding-ada-002") | +| pinecone_id | String | Reference to Pinecone vector ID (indexed) | +| is_synced | Boolean | Sync status flag (indexed) | +| created_at | DateTime | Creation timestamp (indexed) | +| updated_at | DateTime | Last update timestamp | +| last_synced_at | DateTime | Last successful sync to Pinecone | + +**Relationships:** +- Many-to-One with `Chunk` + +**Indexes:** +- `pinecone_id`: For Pinecone reference lookups +- `is_synced`: For finding unsynced embeddings + +## Setup Instructions + +### 1. Prerequisites + +- Python 3.9+ +- SQLite (included with Python) +- Optional: PostgreSQL for production + +### 2. Installation + +```bash +# Install dependencies +pip install -r requirements.txt + +# Or with pip tools +pip-sync requirements.txt +``` + +### 3. Environment Configuration + +Create a `.env` file from the template: + +```bash +cp .env.example .env +``` + +Update with your Pinecone credentials: + +```env +PINECONE_API_KEY=your_api_key_here +PINECONE_ENVIRONMENT=us-east-1 +PINECONE_INDEX_NAME=asktemoc +``` + +### 4. Database Initialization + +Database tables are automatically created on application startup via the `init_db()` function in `app.main`. + +To manually initialize: + +```python +from app.db.database import init_db +init_db() +``` + +## API Endpoints + +### Document Management + +#### Create Document +``` +POST /api/documents +Content-Type: application/json + +{ + "title": "Document Title", + "source": "https://example.com/doc", + "metadata": {"author": "John Doe"} +} +``` + +#### List Documents +``` +GET /api/documents?skip=0&limit=100&include_deleted=false +``` + +#### Get Document Details +``` +GET /api/documents/{doc_id} +``` + +#### Update Document +``` +PUT /api/documents/{doc_id} +Content-Type: application/json + +{ + "title": "Updated Title", + "metadata": {"updated": true} +} +``` + +#### Delete Document +``` +DELETE /api/documents/{doc_id}?hard_delete=false +``` + +### Chunk Management + +#### Create Chunk +``` +POST /api/documents/{doc_id}/chunks +Content-Type: application/json + +{ + "chunk_index": 0, + "text": "Chunk content here...", + "metadata": {"source_page": 1} +} +``` + +#### Batch Create Chunks +``` +POST /api/documents/{doc_id}/chunks/batch +Content-Type: application/json + +{ + "chunks": [ + { + "chunk_index": 0, + "text": "First chunk", + "metadata": {} + }, + { + "chunk_index": 1, + "text": "Second chunk", + "metadata": {} + } + ] +} +``` + +#### List Document Chunks +``` +GET /api/documents/{doc_id}/chunks?skip=0&limit=1000 +``` + +#### Get Chunk Details +``` +GET /api/chunks/{chunk_id} +``` + +#### Update Chunk +``` +PUT /api/chunks/{chunk_id} +Content-Type: application/json + +{ + "text": "Updated chunk text", + "metadata": {"updated": true} +} +``` + +### Embedding Management + +#### Create Embedding +``` +POST /api/documents/chunks/{chunk_id}/embeddings +Content-Type: application/json + +{ + "vector": [0.1, 0.2, 0.3, ...], + "model": "text-embedding-ada-002" +} +``` + +#### Get Embedding +``` +GET /api/embeddings/{embedding_id} +``` + +#### Update Embedding +``` +PUT /api/embeddings/{embedding_id} +Content-Type: application/json + +{ + "vector": [0.1, 0.2, 0.3, ...], + "pinecone_id": "vector-123" +} +``` + +### Pinecone Export Operations + +#### Export Document Embeddings +``` +POST /api/pinecone/export/document/{doc_id} +``` + +Exports all embeddings for a specific document to Pinecone and updates sync status. + +#### Export Unsynced Embeddings +``` +POST /api/pinecone/export/unsynced?batch_size=100 +``` + +Exports all embeddings not yet synced to Pinecone. + +#### Batch Export Embeddings +``` +POST /api/pinecone/export/batch +Content-Type: application/json + +{ + "embedding_ids": ["emb-1", "emb-2", "emb-3"] +} +``` + +#### Delete Vectors from Pinecone +``` +DELETE /api/pinecone/vectors +Content-Type: application/json + +{ + "vector_ids": ["vec-1", "vec-2"] +} +``` + +#### Get Index Statistics +``` +GET /api/pinecone/index/stats +``` + +Returns Pinecone index statistics including dimension, vector count, and more. + +### Dashboard & Analytics + +#### Dashboard Overview +``` +GET /api/dashboard/overview +``` + +Returns comprehensive dashboard statistics with document list and sync status. + +#### Document Statistics +``` +GET /api/dashboard/document/{doc_id}/stats +``` + +Returns detailed statistics for a specific document. + +#### Export Document as JSON +``` +GET /api/dashboard/document/{doc_id}/export +``` + +Exports document with all chunks and embeddings in JSON format. + +#### Duplicate Document +``` +POST /api/dashboard/document/{doc_id}/duplicate?new_title=Copy+of+Document +``` + +Creates a complete copy of document with all chunks and embeddings. + +#### Batch Delete Documents +``` +POST /api/dashboard/documents/batch-delete?hard_delete=false +Content-Type: application/json + +{ + "doc_ids": ["doc-1", "doc-2", "doc-3"] +} +``` + +#### Search Content +``` +GET /api/dashboard/search?query=search_term&limit=100 +``` + +Searches across all chunks and document titles. + +#### Recent Activity +``` +GET /api/dashboard/activity?days=7&limit=100 +``` + +Returns recent changes to documents, chunks, and embeddings. + +#### Sync Status +``` +GET /api/dashboard/sync-status +``` + +Returns overall sync status and statistics. + +## Service Layer + +### DocumentService +High-level CRUD operations for documents: +- `create_document()`: Create new document +- `get_document()`: Retrieve document by ID +- `list_documents()`: List with pagination +- `update_document()`: Update document fields +- `delete_document()`: Soft or hard delete +- `search_documents()`: Search by title/source + +### ChunkService +High-level CRUD operations for chunks: +- `create_chunk()`: Create chunk within document +- `get_chunk()`: Retrieve chunk by ID +- `list_chunks_by_document()`: Get all chunks for document +- `update_chunk()`: Update chunk content/metadata +- `delete_chunk()`: Soft or hard delete +- `get_chunks_by_ids()`: Retrieve multiple chunks + +### EmbeddingService +High-level CRUD operations for embeddings: +- `create_embedding()`: Create embedding for chunk +- `get_embedding()`: Retrieve by ID +- `get_embedding_by_chunk()`: Get embedding for specific chunk +- `list_unsynced_embeddings()`: Get embeddings needing Pinecone sync +- `update_embedding()`: Update vector/sync status +- `mark_synced()`: Mark as synced with Pinecone +- `get_embeddings_by_document()`: Get all embeddings for document + +### PineconeExportService +Manages Pinecone synchronization: +- `prepare_vectors_for_upsert()`: Format vectors with metadata +- `upsert_vectors()`: Sync to Pinecone with metadata +- `export_document_embeddings()`: Export specific document +- `export_unsynced_embeddings()`: Batch export unsynced +- `delete_from_pinecone()`: Remove vectors +- `search_pinecone()`: Query Pinecone index +- `get_index_stats()`: Retrieve index statistics + +### DocumentManagementUtils +High-level utilities for dashboard and batch operations: +- `get_document_statistics()`: Comprehensive document stats +- `get_all_documents_dashboard()`: Dashboard view of all documents +- `batch_delete_documents()`: Delete multiple documents +- `duplicate_document_with_chunks()`: Clone document with data +- `export_document_to_json()`: Export as JSON +- `search_content_across_documents()`: Global content search +- `get_sync_status_summary()`: Overall sync statistics +- `get_recent_activity()`: Activity tracking + +## Usage Examples + +### Python Usage + +```python +from app.db.database import SessionLocal, get_db +from app.db.services import DocumentService, ChunkService, EmbeddingService +from app.services.pinecone_service import PineconeExportService +from app.services.document_management import DocumentManagementUtils + +# Get database session +db = SessionLocal() + +# Create document +doc = DocumentService.create_document( + db=db, + title="My Document", + source="https://example.com", + metadata={"author": "John"} +) + +# Create chunks +chunk1 = ChunkService.create_chunk( + db=db, + document_id=doc.id, + chunk_index=0, + text="First chunk content...", + metadata={"page": 1} +) + +chunk2 = ChunkService.create_chunk( + db=db, + document_id=doc.id, + chunk_index=1, + text="Second chunk content...", + metadata={"page": 2} +) + +# Create embeddings +import numpy as np + +embedding1 = EmbeddingService.create_embedding( + db=db, + chunk_id=chunk1.id, + vector=np.random.randn(1536).tolist(), + model="text-embedding-ada-002" +) + +# Export to Pinecone +pinecone_svc = PineconeExportService() +result = pinecone_svc.export_document_embeddings(db=db, document_id=doc.id) +print(f"Exported {result['upserted_count']} embeddings") + +# Get statistics +stats = DocumentManagementUtils.get_document_statistics(db=db, doc_id=doc.id) +print(f"Document has {stats['chunk_count']} chunks") +print(f"Sync status: {stats['sync_percentage']}%") + +db.close() +``` + +### FastAPI Usage + +```bash +# Start server +uvicorn app.main:app --reload + +# Create document +curl -X POST http://localhost:8000/api/documents \ + -H "Content-Type: application/json" \ + -d '{"title":"Test","source":"http://example.com"}' + +# Get dashboard overview +curl http://localhost:8000/api/dashboard/overview + +# Export embeddings to Pinecone +curl -X POST http://localhost:8000/api/pinecone/export/unsynced +``` + +## Performance Considerations + +### Indexing Strategy +- Documents are indexed by `created_at` for timeline queries +- Chunks use composite index `(document_id, chunk_index)` for fast retrieval +- Embeddings indexed by `pinecone_id` and `is_synced` for efficient sync operations + +### Batch Operations +- Use `batch_create_chunks` for creating multiple chunks +- Use `export_unsynced_embeddings` with configurable batch size +- Pagination available on list endpoints (default limit: 100-1000) + +### Soft Deletes +- All deletes are soft by default (marks `is_deleted=True`) +- Hard delete option available for permanent removal +- Soft deleted records are excluded from queries unless explicitly included + +## Error Handling + +All endpoints return appropriate HTTP status codes: +- `201 Created`: Successful creation +- `200 OK`: Successful retrieval/update +- `204 No Content`: Successful deletion +- `400 Bad Request`: Invalid input +- `404 Not Found`: Resource not found +- `500 Internal Server Error`: Server-side error + +## Environment Variables + +| Variable | Required | Default | Description | +|----------|----------|---------|-------------| +| DATABASE_URL | No | sqlite:///./asktemoc.db | Database connection string | +| DB_ECHO | No | false | Enable SQL logging | +| PINECONE_API_KEY | Yes | - | Pinecone API key | +| PINECONE_ENVIRONMENT | No | us-east-1 | Pinecone region | +| PINECONE_INDEX_NAME | No | asktemoc | Pinecone index name | + +## Database Migration + +To switch from SQLite to PostgreSQL: + +1. Update `.env`: +```env +DATABASE_URL=postgresql://user:password@localhost/asktemoc +``` + +2. SQLAlchemy will automatically use the appropriate dialect + +## Testing + +```python +# Test document creation +def test_create_document(db): + doc = DocumentService.create_document( + db=db, + title="Test Doc", + source="test://doc" + ) + assert doc.id is not None + assert doc.title == "Test Doc" + +# Test chunk creation +def test_create_chunk(db, document): + chunk = ChunkService.create_chunk( + db=db, + document_id=document.id, + chunk_index=0, + text="Test chunk" + ) + assert chunk.document_id == document.id +``` + +## Troubleshooting + +### Issue: "PINECONE_API_KEY environment variable is required" +**Solution:** Ensure `.env` file has `PINECONE_API_KEY` set before initializing PineconeExportService. + +### Issue: Database locked (SQLite) +**Solution:** Use WAL mode or switch to PostgreSQL for concurrent access: +```python +# In database.py +engine = create_engine( + "sqlite:///asktemoc.db", + connect_args={"timeout": 30}, +) +``` + +### Issue: Large embeddings slow to query +**Solution:** Use pagination and consider indexing on `is_synced` for unsynced queries. + +## Future Enhancements + +- [ ] Bulk embedding import/export +- [ ] Vector dimension auto-detection +- [ ] Pinecone namespace management +- [ ] Advanced filtering and full-text search +- [ ] Document versioning +- [ ] Audit logging +- [ ] Rate limiting +- [ ] Authentication/Authorization diff --git a/backend/Dockerfile b/backend/Dockerfile new file mode 100644 index 0000000..b6c20f2 --- /dev/null +++ b/backend/Dockerfile @@ -0,0 +1,28 @@ +FROM python:3.11-slim + +ENV PYTHONUNBUFFERED=1 \ + PYTHONDONTWRITEBYTECODE=1 \ + PIP_NO_CACHE_DIR=1 \ + PIP_DISABLE_PIP_VERSION_CHECK=1 \ + PIP_ROOT_USER_ACTION=ignore + +WORKDIR /app + +# System deps (if you add more heavy libs later, extend this) +RUN apt-get update && apt-get install -y --no-install-recommends \ + build-essential \ + && rm -rf /var/lib/apt/lists/* + +# Install the lean runtime set (scraping deps live in requirements-scrape.txt). +COPY requirements.txt /app/requirements.txt +RUN pip install --upgrade pip && pip install -r /app/requirements.txt + +# Copy application code +COPY . /app + +# Expose FastAPI default port +EXPOSE 8000 + +# Uvicorn entrypoint +CMD ["uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "8000"] + diff --git a/backend/IMPLEMENTATION_STATUS.md b/backend/IMPLEMENTATION_STATUS.md new file mode 100644 index 0000000..1d8b5ed --- /dev/null +++ b/backend/IMPLEMENTATION_STATUS.md @@ -0,0 +1,316 @@ +# Implementation Status Report + +## ✅ COMPLETE: AskTemoc Backend Database & Vector Export System + +### Summary +Successfully implemented a production-ready SQLite relational database schema with FastAPI REST API endpoints and Pinecone vector export pipeline. The system handles document management, chunk storage, embedding synchronization, and comprehensive dashboard analytics. + +**Implementation Date:** November 13, 2025 +**Total Lines of Code:** 1,788 +**Files Created:** 9 +**Files Modified:** 3 +**API Endpoints:** 40+ +**Database Tables:** 3 +**Service Methods:** 40+ + +--- + +## ✅ Deliverables Checklist + +### 1. Relational Schema Design +- [x] **Document table** - Stores document metadata with soft-delete support +- [x] **Chunk table** - Stores text fragments with sequence ordering +- [x] **Embedding table** - Stores vectors with Pinecone sync tracking +- [x] **Relationships** - Proper foreign keys with cascade delete +- [x] **Indexes** - Composite and individual indexes for performance +- [x] **Metadata columns** - JSON fields for flexible data storage +- [x] **Timestamps** - created_at, updated_at for audit trails +- [x] **Soft deletes** - is_deleted flags throughout + +### 2. Database Service Layer +- [x] **DocumentService** - 8 CRUD + search methods +- [x] **ChunkService** - 7 CRUD + batch methods +- [x] **EmbeddingService** - 10 CRUD + sync methods +- [x] **Database initialization** - Auto-create tables on startup +- [x] **Session management** - FastAPI dependency injection +- [x] **Pagination** - Built into all list endpoints +- [x] **Batch operations** - Efficient bulk imports + +### 3. FastAPI REST Endpoints +- [x] **Document endpoints** - 6 endpoints (CRUD + search) +- [x] **Chunk endpoints** - 6 endpoints (CRUD + batch) +- [x] **Embedding endpoints** - 4 endpoints (CRUD) +- [x] **Pinecone export** - 6 endpoints (sync, search, stats) +- [x] **Dashboard analytics** - 8 endpoints (overview, stats, export) +- [x] **Error handling** - Proper HTTP status codes +- [x] **Input validation** - Pydantic schemas +- [x] **Total: 40+ endpoints** + +### 4. Pinecone Export Pipeline +- [x] **Vector preparation** - Format vectors with rich metadata +- [x] **Batch upsert** - Send vectors to Pinecone +- [x] **Metadata enrichment** - Include document/chunk context +- [x] **Sync tracking** - Update DB with sync status +- [x] **Unsynced queries** - Efficiently find unsynced embeddings +- [x] **Vector deletion** - Remove from Pinecone +- [x] **Index statistics** - Retrieve Pinecone metrics +- [x] **Vector search** - Query Pinecone with embeddings + +### 5. Document Management Utilities +- [x] **Document statistics** - Comprehensive metrics per document +- [x] **Dashboard overview** - All documents with sync status +- [x] **Batch delete** - Soft/hard delete multiple documents +- [x] **Document duplication** - Clone with all chunks/embeddings +- [x] **JSON export** - Export document data for backup +- [x] **Content search** - Search across all chunks +- [x] **Activity tracking** - Recent changes timeline +- [x] **Sync status** - Overall synchronization metrics + +### 6. Configuration & Documentation +- [x] **Environment template** - .env.example with all options +- [x] **API documentation** - 400+ line comprehensive guide +- [x] **Quick start guide** - Setup and usage instructions +- [x] **Schema documentation** - Detailed table/column descriptions +- [x] **Usage examples** - Python and curl examples +- [x] **Error handling** - Status codes and messages +- [x] **Performance guide** - Optimization recommendations + +--- + +## 📁 File Structure + +``` +app/ +├── db/ # Database layer +│ ├── __init__.py # Module exports +│ ├── models.py # ORM models (Document, Chunk, Embedding) +│ ├── database.py # Connection & session management +│ └── services.py # CRUD service classes (40+ methods) +│ +├── api/endpoints/ +│ ├── documents.py # Document/Chunk/Embedding endpoints (14 endpoints) +│ ├── pinecone.py # Pinecone sync/export endpoints (6 endpoints) +│ ├── dashboard.py # Analytics/dashboard endpoints (8 endpoints) +│ ├── query.py # (existing) Query endpoints +│ └── __init__.py +│ +├── services/ +│ ├── pinecone_service.py # Pinecone export operations +│ ├── document_management.py # Dashboard utilities +│ └── (other services) +│ +├── schemas/ +│ ├── db_schemas.py # Pydantic request/response models (20+) +│ └── __init__.py +│ +└── main.py # FastAPI app with all routers + +config/ +├── .env.example # Environment configuration +└── asktemoc.db # SQLite database (auto-created) +``` + +--- + +## 🗄️ Database Schema + +### Three-Table Relational Design + +**Documents** +- Primary key: id (UUID) +- Fields: title, source, doc_metadata (JSON), timestamps, is_deleted +- Indexes: id, created_at, is_deleted + +**Chunks** +- Primary key: id (UUID) +- Fields: document_id (FK), chunk_index, text, chunk_metadata (JSON), timestamps, is_deleted +- Indexes: document_id, (document_id, chunk_index) composite + +**Embeddings** +- Primary key: id (UUID) +- Fields: chunk_id (FK), vector (JSON), model, pinecone_id, is_synced, timestamps +- Indexes: chunk_id, pinecone_id, is_synced + +**Relationships:** +- Document → Chunks (1:N, cascade delete) +- Chunk → Embeddings (1:N, cascade delete) + +--- + +## 🔌 API Overview + +### Document Management (`/api/documents`) +``` +POST /documents - Create document +GET /documents - List documents (paginated) +GET /documents/{id} - Get document details +PUT /documents/{id} - Update document +DELETE /documents/{id} - Delete document +POST /documents/search - Search documents +POST /documents/{id}/chunks - Create chunk +POST /documents/{id}/chunks/batch - Batch create chunks +GET /documents/{id}/chunks - List document chunks +``` + +### Embeddings Management +``` +POST /chunks/{id}/embeddings - Create embedding +GET /embeddings/{id} - Get embedding +PUT /embeddings/{id} - Update embedding +DELETE /embeddings/{id} - Delete embedding +``` + +### Pinecone Export (`/api/pinecone`) +``` +POST /export/document/{id} - Export document embeddings +POST /export/unsynced - Batch export unsynced embeddings +POST /export/batch - Export specific embeddings +DELETE /vectors - Delete from Pinecone +GET /index/stats - Get Pinecone stats +GET /search - Search Pinecone +``` + +### Dashboard (`/api/dashboard`) +``` +GET /overview - Dashboard overview +GET /document/{id}/stats - Document statistics +GET /document/{id}/export - Export as JSON +POST /document/{id}/duplicate - Duplicate document +POST /documents/batch-delete - Batch delete documents +GET /search - Global content search +GET /activity - Recent activity +GET /sync-status - Sync statistics +``` + +--- + +## 🚀 Quick Start + +### 1. Setup Environment +```bash +cp .env.example .env +# Edit .env with Pinecone credentials +``` + +### 2. Install Dependencies +```bash +pip install -r requirements.txt +``` + +### 3. Run Application +```bash +uvicorn app.main:app --reload +``` + +Database automatically initializes on startup. + +### 4. Access +- API: `http://localhost:8000` +- Swagger UI: `http://localhost:8000/docs` +- ReDoc: `http://localhost:8000/redoc` + +--- + +## ✨ Key Features + +### Database Layer +✅ SQLAlchemy ORM with relationship management +✅ SQLite (default) + PostgreSQL support +✅ Soft deletes for data safety +✅ JSON metadata for flexibility +✅ Transaction support +✅ Connection pooling +✅ Comprehensive indexing + +### API Layer +✅ 40+ REST endpoints +✅ Pydantic validation +✅ Error handling with proper HTTP codes +✅ Batch operations for performance +✅ Pagination support +✅ Dependency injection + +### Service Layer +✅ 40+ service methods +✅ Document/Chunk/Embedding management +✅ Pinecone synchronization +✅ Dashboard analytics +✅ Search functionality +✅ Activity tracking + +### Features +✅ Document management with metadata +✅ Text chunking with sequence ordering +✅ Embedding storage and sync +✅ Pinecone vector database integration +✅ Batch import/export +✅ Document duplication +✅ Content search +✅ Sync status tracking +✅ Activity monitoring +✅ Dashboard analytics + +--- + +## 📊 Statistics + +| Metric | Value | +|--------|-------| +| Lines of Code | 1,788 | +| Files Created | 9 | +| API Endpoints | 40+ | +| Database Tables | 3 | +| Service Methods | 40+ | +| Pydantic Models | 20+ | +| Indexes | 6 | +| Documentation Pages | 3 | +| Python Compilation | ✅ Success | + +--- + +## ✅ Verification + +All components verified and working: +- ✅ Database layer imports successfully +- ✅ All ORM models properly defined +- ✅ Service classes instantiate correctly +- ✅ FastAPI app initializes with 35+ routes +- ✅ All Python files compile without errors +- ✅ Pydantic schemas validate correctly +- ✅ No reserved name conflicts + +--- + +## 📚 Documentation + +- **`DATABASE_API_DOCS.md`** - Comprehensive 400+ line API documentation +- **`QUICK_START.md`** - Quick reference and setup guide +- **`IMPLEMENTATION_SUMMARY.md`** - This file +- Inline code documentation - Every class and method documented + +--- + +## 🎯 Next Steps (Optional) + +1. **Authentication/Authorization** - Add user management +2. **Caching** - Redis for performance +3. **Monitoring** - Prometheus/Grafana integration +4. **Testing** - Unit and integration tests +5. **CI/CD** - GitHub Actions workflow +6. **Rate Limiting** - API rate limiting +7. **Backup** - Database backup strategy +8. **Vector Dimensions** - Auto-detect from embeddings + +--- + +## 📞 Support + +For detailed API documentation: See `DATABASE_API_DOCS.md` +For quick setup: See `QUICK_START.md` +For implementation details: See `IMPLEMENTATION_SUMMARY.md` + +--- + +**Status: ✅ PRODUCTION READY** + +All requirements completed. System is ready for deployment with optional enhancements available. diff --git a/backend/IMPLEMENTATION_SUMMARY.md b/backend/IMPLEMENTATION_SUMMARY.md new file mode 100644 index 0000000..47ec4d7 --- /dev/null +++ b/backend/IMPLEMENTATION_SUMMARY.md @@ -0,0 +1,410 @@ +# Implementation Summary: AskTemoc Backend Database & API Layer + +## Overview + +Successfully implemented a complete SQLite relational schema with FastAPI endpoints for document management, embeddings storage, and ChromaDB export pipeline. The system is production-ready with comprehensive CRUD operations, batch utilities, and dashboard analytics. + +**Total Code Added:** 1,788 lines of well-documented, tested Python code + +## Implementation Complete ✓ + +### What Was Built + +1. **Relational Database Schema** (3 tables with relationships) + - Documents table with metadata storage + - Chunks table with sequence ordering + - Embeddings table with Pinecone sync tracking + - Proper indexes for query performance + - Soft delete support throughout + +2. **Database Service Layer** (40+ methods) + - Full CRUD operations for all entities + - Search and filtering capabilities + - Batch operations support + - Relationship management + +3. **FastAPI Endpoints** (30+ endpoints) + - Complete REST API for all operations + - Batch import/export capabilities + - Proper error handling and status codes + - Input validation with Pydantic + +4. **ChromaDB Export Pipeline** + - Vector upsert with metadata + - Batch sync operations + - Sync status tracking + - Vector deletion support + - Index statistics + +5. **Dashboard & Analytics** + - Document statistics and overview + - Batch operations (delete, duplicate) + - Global content search + - Activity tracking + - Sync status monitoring + +## Files Created/Modified + +### Core Database Layer +1. **`app/db/models.py`** (NEW) + - `Document`: Document metadata storage with soft-delete + - `Chunk`: Text fragments with sequence ordering + - `Embedding`: Vector storage with Pinecone sync tracking + - Relationships: Document → Chunks → Embeddings + +2. **`app/db/database.py`** (NEW) + - SQLAlchemy engine and session factory + - Database initialization (`init_db()`) + - Dependency injection for FastAPI (`get_db()`) + - Support for SQLite and PostgreSQL + +3. **`app/db/services.py`** (NEW) + - `DocumentService`: CRUD + search for documents + - `ChunkService`: CRUD + batch operations for chunks + - `EmbeddingService`: CRUD + sync tracking for embeddings + - 40+ methods for database operations + +4. **`app/db/__init__.py`** (NEW) + - Module exports for clean imports + +### Service Layer +5. **`app/services/chroma_service.py`** (NEW) + - `ChromaService`: Vector upsert, metadata sync + - Prepare vectors with rich metadata + - Batch export operations + - Vector search and deletion + - Index statistics retrieval + +6. **`app/services/document_management.py`** (NEW) + - `DocumentManagementUtils`: High-level dashboard utilities + - Document statistics and overview + - Batch operations (delete, duplicate) + - JSON export functionality + - Global content search + - Sync status tracking + - Activity monitoring + +### API Endpoints +7. **`app/api/endpoints/documents.py`** (NEW) + - Document CRUD: POST, GET, PUT, DELETE + - Chunk CRUD with batch operations + - Embedding CRUD + - Full routing for document hierarchy + +8. **`app/api/endpoints/chroma.py`** (NEW) + - Export endpoints for ChromaDB sync + - Document-specific export + - Batch and unsynced export + - Vector deletion + - Index statistics + - Search integration + +9. **`app/api/endpoints/dashboard.py`** (NEW) + - Dashboard overview + - Document statistics + - Document export/duplicate + - Batch operations + - Content search + - Activity tracking + - Sync status + +### Configuration & Documentation +10. **`app/schemas/db_schemas.py`** (NEW) + - 20+ Pydantic models for request/response validation + - Type-safe API contracts + - Comprehensive documentation + +11. **`app/main.py`** (MODIFIED) + - Added startup event for database initialization + - Integrated all new routers + - Organized by feature (documents, chroma, dashboard) + +12. **`requirements.txt`** (MODIFIED) + - Added `chromadb` + - SQLAlchemy already present + +13. **`.env.example`** (NEW) + - Environment configuration template + - ChromaDB settings + - Database options + +14. **`DATABASE_API_DOCS.md`** (NEW) + - Comprehensive 400+ line documentation + - Schema design details + - API endpoint specifications + - Usage examples + - Performance considerations + - Troubleshooting guide + +15. **`QUICK_START.md`** (NEW) + - Quick reference guide + - File structure overview + - Common workflows + - Setup instructions + +## Database Schema + +### Relational Structure +``` +documents (1) + ├── chunks (*) + │ └── embeddings (*) +``` + +### Key Features +- **Soft Deletes**: `is_deleted` flag prevents accidental data loss +- **Metadata Storage**: JSON columns for flexible data +- **Timestamps**: `created_at`, `updated_at` for audit trails +- **Sync Tracking**: `is_synced`, `chroma_id` for ChromaDB management +- **Indexing**: Composite and individual indexes for performance + +### Database Indexes +- `documents.created_at`, `is_deleted` +- `chunks.(document_id, chunk_index)` - Composite index +- `chunks.is_deleted` +- `embeddings.chroma_id`, `is_synced` + +## API Endpoints (40+) + +### Documents (`/api/documents` - 7 endpoints) +- POST, GET, GET/{id}, PUT/{id}, DELETE/{id} +- POST /search +- Total: 6 endpoints + +### Chunks (`/api/documents/{id}/chunks` - 8 endpoints) +- POST, POST /batch, GET +- Individual chunk operations: GET, PUT, DELETE +- Total: 6 endpoints + +### Embeddings (`/api/documents/chunks/{id}/embeddings` - 5 endpoints) +- POST, GET/{id}, PUT/{id}, DELETE/{id} +- Total: 4 endpoints + +### ChromaDB (`/api/chroma` - 6 endpoints) +- POST /export/document/{id} +- POST /export/unsynced +- POST /export/batch +- DELETE /vectors +- GET /index/stats +- GET /search +- Total: 6 endpoints + +### Dashboard (`/api/dashboard` - 8 endpoints) +- GET /overview +- GET /document/{id}/stats +- GET /document/{id}/export +- POST /document/{id}/duplicate +- POST /documents/batch-delete +- GET /search +- GET /activity +- GET /sync-status +- Total: 8 endpoints + +## Key Features Implemented + +### 1. Relational Schema ✅ +- Three-table design: Documents → Chunks → Embeddings +- Foreign key relationships with cascade deletes +- Support for metadata at each level +- Soft deletes for data safety + +### 2. Complete CRUD Operations ✅ +- Create, Read, Update, Delete for all entities +- Batch creation for chunks +- Soft and hard delete options +- Search and filtering capabilities + +### 3. ChromaDB Integration ✅ +- Vector metadata preparation with document/chunk context +- Batch upsert to ChromaDB +- Sync status tracking +- Automatic chroma_id assignment +- Vector deletion capability +- Vector search integration + +### 4. Document Management Utilities ✅ +- Document statistics and metrics +- Dashboard overview with sync status +- Batch delete operations +- Document duplication with data +- JSON export for backup +- Global content search +- Activity tracking + +### 5. Dashboard Analytics ✅ +- Overview statistics (documents, chunks, embeddings) +- Sync status percentage +- Recent activity timeline +- Document-level statistics +- Content search across all documents +- Document management (duplicate, batch delete) + +### 6. Error Handling ✅ +- Proper HTTP status codes (201, 204, 400, 404, 500) +- Meaningful error messages +- Graceful failure handling +- Exception propagation + +## Configuration + +### Environment Variables +``` +DATABASE_URL # SQLite or PostgreSQL +DB_ECHO # SQL logging (true/false) +CHROMA_PERSIST_DIRECTORY # Directory for ChromaDB persistence +CHROMA_COLLECTION_NAME # Collection name +``` + +### Database Support +- **SQLite**: Default, no additional setup +- **PostgreSQL**: Set DATABASE_URL environment variable + +## Service Layer Architecture + +### Layering +``` +FastAPI Routes (app/api/endpoints/) + ↓ +Pydantic Schemas (app/schemas/) + ↓ +Business Logic Services (app/services/) + ↓ +Database Services (app/db/services.py) + ↓ +SQLAlchemy Models (app/db/models.py) + ↓ +Database Engine (app/db/database.py) +``` + +### Service Responsibilities +- **DocumentService**: Document lifecycle +- **ChunkService**: Chunk management and organization +- **EmbeddingService**: Embedding storage and sync +- **ChromaService**: External vector DB synchronization +- **DocumentManagementUtils**: High-level business operations + +## Usage Patterns + +### Pattern 1: Document Ingestion +```python +1. Create document +2. Create chunks (batch) +3. Generate embeddings +4. Export to Pinecone +5. Update sync status +``` + +### Pattern 2: Dashboard Access +```python +1. Get all documents with stats +2. Calculate sync percentages +3. Track recent activity +4. Render dashboard UI +``` + +### Pattern 3: Content Search +```python +1. Search documents by title/source +2. Search chunks by text content +3. Combine results +4. Return ranked results +``` + +### Pattern 4: ChromaDB Sync +```python +1. Find unsynced embeddings +2. Prepare vectors with metadata +3. Batch upsert to ChromaDB +4. Update sync status in DB +``` + +## Production Readiness + +### Included Features +✅ Soft deletes for data safety +✅ Transaction support via SQLAlchemy +✅ Connection pooling +✅ Comprehensive error handling +✅ Request validation (Pydantic) +✅ Batch operations for performance +✅ Pagination support +✅ Full audit trail (timestamps) +✅ Sync tracking for external systems +✅ Flexible metadata storage (JSON) + +### Recommended for Production +- [ ] Authentication/Authorization +- [ ] Rate limiting +- [ ] Request logging/monitoring +- [ ] Database backups +- [ ] Connection pooling optimization +- [ ] Query performance monitoring +- [ ] Cache layer (Redis) +- [ ] API versioning +- [ ] Comprehensive testing +- [ ] CI/CD integration + +## Performance Considerations + +### Indexing +- Composite index on `(document_id, chunk_index)` for chunk retrieval +- Individual indexes on frequently filtered columns +- Pinecone ID indexed for reference lookups + +### Query Optimization +- Pagination built into list endpoints +- Lazy loading relationships +- Batch operations support +- Soft delete filters applied automatically + +### Scalability +- Support for PostgreSQL with connection pooling +- Batch export to handle large document sets +- Efficient unsynced embedding queries +- Composite indexing for nested queries + +## Testing + +All Python files compile successfully without syntax errors. Ready for unit and integration testing. + +## Documentation + +### Comprehensive Guides +1. **DATABASE_API_DOCS.md** - Full API reference (400+ lines) + - Schema design + - All endpoints + - Service documentation + - Usage examples + - Error handling + - Troubleshooting + +2. **QUICK_START.md** - Quick reference guide + - File structure + - Quick setup + - Common workflows + - Service signatures + - Environment config + +## Next Steps + +1. **Integration**: Connect embedding generation service +2. **Testing**: Add unit and integration tests +3. **Security**: Implement authentication/authorization +4. **Monitoring**: Add logging and monitoring +5. **Optimization**: Performance tuning based on actual usage +6. **Features**: Document versioning, advanced search, caching + +## Summary + +This implementation provides a production-ready database and API layer for the AskTemoc backend with: + +- ✅ **Relational Schema**: Documents → Chunks → Embeddings with metadata +- ✅ **Complete CRUD Operations**: Full lifecycle management +- ✅ **ChromaDB Export Pipeline**: Vector sync with metadata storage +- ✅ **Document Management Utilities**: Dashboard and batch operations +- ✅ **FastAPI Integration**: 40+ endpoints, proper validation +- ✅ **Error Handling**: Comprehensive error management +- ✅ **Documentation**: Complete guides and examples +- ✅ **Production Ready**: Soft deletes, timestamps, audit trails + +The system is ready for integration with embedding services and RAG pipelines. diff --git a/backend/QUICK_START.md b/backend/QUICK_START.md new file mode 100644 index 0000000..a9e4e88 --- /dev/null +++ b/backend/QUICK_START.md @@ -0,0 +1,378 @@ +# AskTemoc Backend - Quick Start Guide + +## File Structure + +``` +app/ +├── db/ # Database layer +│ ├── __init__.py # Module exports +│ ├── models.py # SQLAlchemy ORM models (Document, Chunk, Embedding) +│ ├── database.py # Database connection & session management +│ └── services.py # CRUD service classes +│ +├── services/ # Business logic services +│ ├── chroma_service.py # ChromaDB export & sync operations +│ ├── document_management.py # High-level document utilities & dashboard helpers +│ ├── rag_service.py # (existing) RAG operations +│ └── ...other services +│ +├── api/endpoints/ # FastAPI route handlers +│ ├── documents.py # Document/Chunk/Embedding CRUD endpoints +│ ├── chroma.py # ChromaDB sync/export endpoints +│ ├── dashboard.py # Analytics & dashboard endpoints +│ ├── query.py # (existing) Query endpoints +│ └── __init__.py +│ +├── models/ # Pydantic schemas (requests/responses) +│ ├── requests.py +│ ├── response.py +│ └── responses.py +│ +├── schemas/ +│ ├── db_schemas.py # NEW: All DB-related Pydantic schemas +│ └── __init__.py +│ +└── main.py # FastAPI app initialization with all routers + +config/ +├── .env.example # Environment configuration template +└── asktemoc.db # SQLite database (auto-created) +``` + +## Key Components + +### 1. Database Models (`app/db/models.py`) +- **Document**: Stores document metadata (title, source, custom metadata) +- **Chunk**: Text fragments extracted from documents with sequence ordering +- **Embedding**: Vector embeddings with ChromaDB sync tracking + +### 2. Service Layer (`app/db/services.py`) +- `DocumentService`: Create, read, update, delete documents +- `ChunkService`: Manage document chunks +- `EmbeddingService`: Handle embeddings and sync status + +### 3. API Endpoints + +#### Document Management (`/api/documents`) +``` +POST /documents - Create document +GET /documents - List documents +GET /documents/{id} - Get document details +PUT /documents/{id} - Update document +DELETE /documents/{id} - Delete document +POST /documents/search - Search documents +``` + +#### Chunks (`/api/documents/{id}/chunks`) +``` +POST /documents/{id}/chunks - Create chunk +POST /documents/{id}/chunks/batch - Batch create chunks +GET /documents/{id}/chunks - List document chunks +PUT /chunks/{id} - Update chunk +DELETE /chunks/{id} - Delete chunk +``` + +#### Embeddings (`/api/documents/chunks/{id}/embeddings`) +``` +POST /chunks/{id}/embeddings - Create embedding +GET /embeddings/{id} - Get embedding +PUT /embeddings/{id} - Update embedding +DELETE /embeddings/{id} - Delete embedding +``` + +#### ChromaDB Export (`/api/chroma`) +``` +POST /export/document/{id} - Export document embeddings +POST /export/unsynced - Export unsynced embeddings +POST /export/batch - Export specific embeddings +DELETE /vectors - Delete from ChromaDB +GET /index/stats - Get index statistics +GET /search - Search ChromaDB +``` + +#### Dashboard (`/api/dashboard`) +``` +GET /overview - Dashboard overview +GET /document/{id}/stats - Document statistics +GET /document/{id}/export - Export as JSON +POST /document/{id}/duplicate - Duplicate document +POST /documents/batch-delete - Batch delete +GET /search - Global content search +GET /activity - Recent activity +GET /sync-status - Sync statistics +``` + +## Quick Start + +### Requirements + +- Python 3.13+ +- [`ollama`](https://ollama.com/) installed and running (for future integration, currently mocked) +- `pip` (Python package installer) +- Git + +--- + +### Clone the Repository + +```bash +git clone https://github.com/Conwenu/AskTemoc_Backend.git +cd path/to/project-root +``` + +--- + +### Install Dependencies + +Make sure you're in a **virtual environment**: + +```bash +python -m venv venv +source venv/bin/activate # On Windows: venv\Scripts\activate +``` + +Install required packages: + +```bash +pip install -r requirements.txt +``` + +--- + +### Make Sure Ollama is Installed + +Ensure [`ollama`](https://ollama.com/) is installed and running locally. + +```bash +ollama run llama3 # Or any other model you plan to use +``` + +--- + +### Run the FastAPI Server + +You can start the server using: + +```bash +uvicorn app.main:app --reload +``` + +- Visit Swagger UI: [http://localhost:8000/docs](http://localhost:8000/docs) + + +### Example: Create Document with Chunks + +```bash +# 1. Create document +DOCUMENT_ID=$(curl -X POST http://localhost:8000/api/documents \ + -H "Content-Type: application/json" \ + -d '{"title":"My Document","source":"https://example.com"}' | jq -r '.id') + +# 2. Create chunks +curl -X POST http://localhost:8000/api/documents/$DOCUMENT_ID/chunks/batch \ + -H "Content-Type: application/json" \ + -d '{ + "chunks": [ + {"chunk_index":0,"text":"First chunk content..."}, + {"chunk_index":1,"text":"Second chunk content..."} + ] + }' + +# 3. View dashboard +curl http://localhost:8000/api/dashboard/overview +``` + +## Database Schema + +### Documents Table +```sql +CREATE TABLE documents ( + id TEXT PRIMARY KEY, + title VARCHAR(255) NOT NULL, + source VARCHAR(512), + metadata JSON, + created_at DATETIME DEFAULT CURRENT_TIMESTAMP, + updated_at DATETIME DEFAULT CURRENT_TIMESTAMP, + is_deleted BOOLEAN DEFAULT 0 +); +``` + +### Chunks Table +```sql +CREATE TABLE chunks ( + id TEXT PRIMARY KEY, + document_id TEXT NOT NULL REFERENCES documents(id), + chunk_index INTEGER NOT NULL, + text TEXT NOT NULL, + metadata JSON, + created_at DATETIME DEFAULT CURRENT_TIMESTAMP, + updated_at DATETIME DEFAULT CURRENT_TIMESTAMP, + is_deleted BOOLEAN DEFAULT 0 +); +CREATE INDEX idx_document_chunk_index ON chunks(document_id, chunk_index); +``` + +### Embeddings Table +```sql +CREATE TABLE embeddings ( + id TEXT PRIMARY KEY, + chunk_id TEXT NOT NULL REFERENCES chunks(id), + vector JSON, + model VARCHAR(100), + chroma_id VARCHAR(255), + is_synced BOOLEAN DEFAULT 0, + created_at DATETIME DEFAULT CURRENT_TIMESTAMP, + updated_at DATETIME DEFAULT CURRENT_TIMESTAMP, + last_synced_at DATETIME +); +CREATE INDEX idx_chroma_id ON embeddings(chroma_id); +CREATE INDEX idx_is_synced ON embeddings(is_synced); +``` + +## Service Classes + +### DocumentService +```python +DocumentService.create_document(db, title, source, metadata) +DocumentService.get_document(db, doc_id) +DocumentService.list_documents(db, skip, limit, include_deleted) +DocumentService.update_document(db, doc_id, title, source, metadata) +DocumentService.delete_document(db, doc_id, hard_delete) +DocumentService.search_documents(db, query_str) +``` + +### ChunkService +```python +ChunkService.create_chunk(db, document_id, chunk_index, text, metadata) +ChunkService.get_chunk(db, chunk_id) +ChunkService.list_chunks_by_document(db, document_id, skip, limit) +ChunkService.update_chunk(db, chunk_id, text, metadata) +ChunkService.delete_chunk(db, chunk_id, hard_delete) +ChunkService.get_chunks_by_ids(db, chunk_ids) +``` + +### EmbeddingService +```python +EmbeddingService.create_embedding(db, chunk_id, vector, model) +EmbeddingService.get_embedding(db, embedding_id) +EmbeddingService.list_unsynced_embeddings(db, limit) +EmbeddingService.mark_synced(db, embedding_id, chroma_id) +EmbeddingService.update_embedding(db, embedding_id, vector, chroma_id, is_synced) +EmbeddingService.delete_embedding(db, embedding_id) +EmbeddingService.get_embeddings_by_document(db, document_id) +``` + +### ChromaService +```python +chroma_svc = ChromaService() +chroma_svc.upsert_vectors(db, embeddings) # Send to ChromaDB +chroma_svc.export_document_embeddings(db, document_id) +chroma_svc.export_unsynced_embeddings(db, batch_size) +chroma_svc.delete_from_chroma(vector_ids) +chroma_svc.search_chroma(query_vector, top_k) +``` + +### DocumentManagementUtils +```python +DocumentManagementUtils.get_document_statistics(db, doc_id) +DocumentManagementUtils.get_all_documents_dashboard(db) +DocumentManagementUtils.batch_delete_documents(db, doc_ids, hard_delete) +DocumentManagementUtils.duplicate_document_with_chunks(db, source_doc_id, new_title) +DocumentManagementUtils.export_document_to_json(db, doc_id) +DocumentManagementUtils.search_content_across_documents(db, search_query, limit) +DocumentManagementUtils.get_sync_status_summary(db) +DocumentManagementUtils.get_recent_activity(db, days, limit) +``` + +## Common Workflows + +### Workflow 1: Ingest Document with Embeddings +```python +# 1. Create document +doc = DocumentService.create_document(db, title, source) + +# 2. Create chunks (batch) +for i, text in enumerate(chunks_text): + chunk = ChunkService.create_chunk(db, doc.id, i, text) + +# 3. Generate and create embeddings +chunks = ChunkService.list_chunks_by_document(db, doc.id) +for chunk in chunks: + vector = embedding_model.encode(chunk.text) + embedding = EmbeddingService.create_embedding(db, chunk.id, vector) + +# 4. Export to ChromaDB +chroma_svc = ChromaService() +result = chroma_svc.export_document_embeddings(db, doc.id) +``` + +### Workflow 2: Query with RAG +```python +# 1. Search ChromaDB with query embedding +query_vector = embedding_model.encode(user_query) +chroma_svc = ChromaService() +results = chroma_svc.search_chroma(query_vector, top_k=5) + +# 2. Retrieve full chunk data from database +for match in results['matches']: + chroma_id = match['id'] + embedding = db.query(Embedding).filter(Embedding.chroma_id == chroma_id).first() + chunk = EmbeddingService.get_embedding(db, embedding.id).chunk + # Use chunk.text in LLM context +``` + +### Workflow 3: Dashboard Overview +```python +overview = DocumentManagementUtils.get_all_documents_dashboard(db) +sync_status = DocumentManagementUtils.get_sync_status_summary(db) +activity = DocumentManagementUtils.get_recent_activity(db, days=7) + +# Returns statistics for UI rendering +``` + +## API Documentation + +Full API documentation available at: `/DATABASE_API_DOCS.md` + +Interactive API docs (Swagger UI): `http://localhost:8000/docs` + +## Environment Variables + +``` +DATABASE_URL SQLite or PostgreSQL connection string +DB_ECHO Enable SQL query logging (true/false) +CHROMA_PERSIST_DIRECTORY Directory for ChromaDB persistence (default: ./app/chroma_db) +CHROMA_COLLECTION_NAME ChromaDB collection name (default: asktemoc) +``` + +## Troubleshooting + +### Database not initializing +- Check DATABASE_URL in .env +- Ensure write permissions in working directory +- For PostgreSQL, verify connection string format + +### ChromaDB export fails +- Verify CHROMA_PERSIST_DIRECTORY is writable +- Check embeddings have valid vectors +- Ensure ChromaDB collection can be created + +### Slow queries +- Use pagination (limit parameter) +- Leverage indexes on (document_id, chunk_index) +- Filter by is_synced for efficient unsynced queries + +## Next Steps + +1. Integrate with embedding generation service +2. Add authentication/authorization +3. Implement rate limiting +4. Add comprehensive logging +5. Set up monitoring and alerting +6. Consider document versioning +7. Implement full-text search +8. Add audit logging + +## Support + +For detailed documentation, see: `/DATABASE_API_DOCS.md` diff --git a/backend/README.md b/backend/README.md index b3cadf2..8045123 100644 --- a/backend/README.md +++ b/backend/README.md @@ -1,61 +1,37 @@ -# UTD Catalog Scraper +# Ask Temoc API -A Python web scraper using Playwright to extract program requirements and example degree plans from the UTD undergraduate catalog. +FastAPI service behind the Ask Temoc web app: a LangChain RAG chain over a +Chroma collection seeded from `data/catalog/`, streamed as server-sent events. +For setup, see the [root README](../README.md). -## Installation +## Layout -1. Install Python dependencies: -```bash -pip install -r requirements.txt -``` - -2. Install Playwright browsers: -```bash -playwright install ``` - -## Usage - -### Basic Usage - -```python -from scraper import UTDCatalogScraper - -scraper = UTDCatalogScraper() -await scraper.scrape() +app/ + main.py app, CORS, routers + core/config.py all settings (env + backend/.env), see .env.example + api/endpoints/rag_endpoint.py POST /api/chat (SSE) + api/endpoints/health.py GET /api/health + services/rag_chain_service.py the RAG chain (built lazily) + services/retriever_service.py Chroma retriever + shared embeddings factory + db/ SQLAlchemy models and services +scripts/seed_catalog.py load data/catalog into Chroma (idempotent) +scrapers/catalog/ Playwright catalog scraper + sources.json builder +tests/ pytest suite (offline) ``` -### With Configuration +## Requirements files -```python -scraper = UTDCatalogScraper( - max_pages=10, # Limit to 10 pages - rate_limit=2.0, # 2 second delay between requests - max_parallel=5, # 5 concurrent browser instances - output_dir="./data" # Custom output directory -) -await scraper.scrape() -``` +| File | For | +| --- | --- | +| `requirements.txt` | running the API and the seed script (the Dockerfile uses it) | +| `requirements-dev.txt` | the above plus pytest | +| `requirements-scrape.txt` | scrapers and the Crawl4AI/docling ingestion experiments | -### Testing with Single Page +## Common commands (from `backend/`, venv active) ```bash -python test_scraper.py -``` - -## Configuration Options - -- `max_pages`: Limit number of pages to scrape (None for all) -- `rate_limit`: Delay between requests in seconds (default: 1.0) -- `max_parallel`: Number of concurrent browser instances (default: 3) -- `output_dir`: Output directory path (default: `./output`) - -## Output Structure - +python scripts/seed_catalog.py # seed / re-seed the vector store +uvicorn app.main:app --reload --port 8000 # run the API +python -m pytest # tests ``` -output/ - {major_name}/ - requirements.txt - example.txt -``` - diff --git a/backend/app/api/__init__.py b/backend/app/api/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/backend/app/api/endpoints/chroma.py b/backend/app/api/endpoints/chroma.py new file mode 100644 index 0000000..82f315c --- /dev/null +++ b/backend/app/api/endpoints/chroma.py @@ -0,0 +1,175 @@ +""" +FastAPI endpoints for ChromaDB export operations. +""" + +from typing import List, Optional +from fastapi import APIRouter, Depends, HTTPException, status +from sqlalchemy.orm import Session + +from app.db.database import get_db +from app.db.services import EmbeddingService, DocumentService +from app.services.chroma_service import ChromaService +from app.schemas.db_schemas import ChromaExportResponse, ChromaIndexStats + +router = APIRouter(prefix="/chroma", tags=["chroma"]) + + +def get_chroma_service(): + """Dependency to get ChromaService.""" + try: + return ChromaService() + except Exception as e: + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail=f"Failed to initialize Chroma service: {str(e)}", + ) + + +@router.post("/export/document/{doc_id}", response_model=ChromaExportResponse) +def export_document_embeddings( + doc_id: str, + db: Session = Depends(get_db), + chroma_svc: ChromaService = Depends(get_chroma_service), +): + """Export all embeddings for a specific document to ChromaDB.""" + document = DocumentService.get_document(db=db, doc_id=doc_id) + if not document: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, detail="Document not found" + ) + + result = chroma_svc.export_document_embeddings(db=db, document_id=doc_id) + + if result.get("status") == "error": + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail=result.get("error"), + ) + + return ChromaExportResponse( + status=result.get("status"), + message=f"Exported {result.get('upserted_count', 0)} embeddings", + upserted_count=result.get("upserted_count"), + updated_db_count=result.get("updated_db_count"), + ) + + +@router.post("/export/unsynced", response_model=ChromaExportResponse) +def export_unsynced_embeddings( + batch_size: int = 100, + db: Session = Depends(get_db), + chroma_svc: ChromaService = Depends(get_chroma_service), +): + """Export all unsynced embeddings to ChromaDB.""" + result = chroma_svc.export_unsynced_embeddings(db=db, batch_size=batch_size) + + if result.get("status") == "error": + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail=result.get("error"), + ) + + return ChromaExportResponse( + status=result.get("status"), + message=f"Exported {result.get('upserted_count', 0)} embeddings", + upserted_count=result.get("upserted_count"), + updated_db_count=result.get("updated_db_count"), + ) + + +@router.post("/export/batch", response_model=ChromaExportResponse) +def export_batch_embeddings( + embedding_ids: List[str], + db: Session = Depends(get_db), + chroma_svc: ChromaService = Depends(get_chroma_service), +): + """Export a batch of specific embeddings to ChromaDB.""" + embeddings = EmbeddingService.get_embeddings_by_ids(db=db, embedding_ids=embedding_ids) + + if not embeddings: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, detail="No embeddings found" + ) + + result = chroma_svc.upsert_vectors(db=db, embeddings=embeddings) + + if result.get("status") == "error": + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail=result.get("error"), + ) + + return ChromaExportResponse( + status=result.get("status"), + message=f"Exported {result.get('upserted_count', 0)} embeddings", + upserted_count=result.get("upserted_count"), + updated_db_count=result.get("updated_db_count"), + ) + + +@router.delete("/vectors", response_model=ChromaExportResponse) +def delete_vectors_from_chroma( + vector_ids: List[str], + chroma_svc: ChromaService = Depends(get_chroma_service), +): + """Delete vectors from ChromaDB.""" + if not vector_ids: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, detail="No vector IDs provided" + ) + + result = chroma_svc.delete_from_chroma(vector_ids=vector_ids) + + if result.get("status") == "error": + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail=result.get("error"), + ) + + return ChromaExportResponse( + status=result.get("status"), + message=f"Deleted {result.get('deleted_count', 0)} vectors", + ) + + +@router.get("/index/stats", response_model=ChromaIndexStats) +def get_index_statistics( + chroma_svc: ChromaService = Depends(get_chroma_service), +): + """Get ChromaDB collection statistics.""" + result = chroma_svc.get_collection_stats() + + if result.get("status") == "error": + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail=result.get("error"), + ) + + return ChromaIndexStats( + status=result.get("status"), + stats=result.get("stats"), + ) + + +@router.get("/search", response_model=dict) +def search_chroma( + query_vector: List[float], + top_k: int = 10, + db: Session = Depends(get_db), + chroma_svc: ChromaService = Depends(get_chroma_service), +): + """Search ChromaDB collection with query vector.""" + if not query_vector: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, detail="Query vector required" + ) + + result = chroma_svc.search_chroma(query_vector=query_vector, top_k=top_k) + + if result.get("status") == "error": + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail=result.get("error"), + ) + + return result diff --git a/backend/app/api/endpoints/dashboard.py b/backend/app/api/endpoints/dashboard.py new file mode 100644 index 0000000..1ea2b49 --- /dev/null +++ b/backend/app/api/endpoints/dashboard.py @@ -0,0 +1,133 @@ +""" +FastAPI endpoints for dashboard and analytics. +""" + +from typing import List, Optional +from fastapi import APIRouter, Depends, HTTPException, status +from sqlalchemy.orm import Session + +from app.db.database import get_db +from app.services.document_management import DocumentManagementUtils +from app.db.services import DocumentService + +router = APIRouter(prefix="/dashboard", tags=["dashboard"]) + + +@router.get("/overview") +def get_dashboard_overview(db: Session = Depends(get_db)): + """Get overall dashboard statistics.""" + sync_status = DocumentManagementUtils.get_sync_status_summary(db=db) + documents = DocumentManagementUtils.get_all_documents_dashboard(db=db) + + return { + "sync_status": sync_status, + "documents": documents, + "total_documents": len(documents), + } + + +@router.get("/document/{doc_id}/stats") +def get_document_stats(doc_id: str, db: Session = Depends(get_db)): + """Get detailed statistics for a specific document.""" + document = DocumentService.get_document(db=db, doc_id=doc_id) + if not document: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, detail="Document not found" + ) + + stats = DocumentManagementUtils.get_document_statistics(db=db, doc_id=doc_id) + return stats + + +@router.get("/document/{doc_id}/export") +def export_document_json(doc_id: str, db: Session = Depends(get_db)): + """Export document with all chunks and embeddings as JSON.""" + document = DocumentService.get_document(db=db, doc_id=doc_id) + if not document: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, detail="Document not found" + ) + + export_data = DocumentManagementUtils.export_document_to_json(db=db, doc_id=doc_id) + return export_data + + +@router.post("/document/{doc_id}/duplicate") +def duplicate_document( + doc_id: str, new_title: Optional[str] = None, db: Session = Depends(get_db) +): + """Duplicate a document with all its chunks and embeddings.""" + document = DocumentService.get_document(db=db, doc_id=doc_id) + if not document: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, detail="Document not found" + ) + + new_doc = DocumentManagementUtils.duplicate_document_with_chunks( + db=db, source_doc_id=doc_id, new_title=new_title + ) + + if not new_doc: + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail="Failed to duplicate document", + ) + + return { + "status": "success", + "original_id": doc_id, + "duplicate_id": new_doc.id, + "duplicate_title": new_doc.title, + } + + +@router.post("/documents/batch-delete") +def batch_delete_documents( + doc_ids: List[str], hard_delete: bool = False, db: Session = Depends(get_db) +): + """Batch delete multiple documents.""" + if not doc_ids: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, detail="No document IDs provided" + ) + + result = DocumentManagementUtils.batch_delete_documents( + db=db, doc_ids=doc_ids, hard_delete=hard_delete + ) + + return result + + +@router.get("/search") +def search_content(query: str, limit: int = 100, db: Session = Depends(get_db)): + """Search for content across all documents.""" + if not query: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, detail="Search query required" + ) + + results = DocumentManagementUtils.search_content_across_documents( + db=db, search_query=query, limit=limit + ) + + return { + "query": query, + "result_count": len(results), + "results": results, + } + + +@router.get("/activity") +def get_recent_activity(days: int = 7, limit: int = 100, db: Session = Depends(get_db)): + """Get recent activity across documents, chunks, and embeddings.""" + activity = DocumentManagementUtils.get_recent_activity( + db=db, days=days, limit=limit + ) + return activity + + +@router.get("/sync-status") +def get_sync_status(db: Session = Depends(get_db)): + """Get current sync status with ChromaDB.""" + sync_status = DocumentManagementUtils.get_sync_status_summary(db=db) + return sync_status diff --git a/backend/app/api/endpoints/documents.py b/backend/app/api/endpoints/documents.py new file mode 100644 index 0000000..b12fe75 --- /dev/null +++ b/backend/app/api/endpoints/documents.py @@ -0,0 +1,287 @@ +""" +FastAPI endpoints for document management. +""" + +from typing import List, Optional +from fastapi import APIRouter, Depends, HTTPException, status +from sqlalchemy.orm import Session + +from app.db.database import get_db +from app.db.services import DocumentService, ChunkService, EmbeddingService +from app.schemas.db_schemas import ( + DocumentCreate, + DocumentUpdate, + DocumentResponse, + DocumentDetailResponse, + DocumentSearch, + ChunkCreate, + ChunkUpdate, + ChunkResponse, + ChunkDetailResponse, + EmbeddingCreate, + EmbeddingUpdate, + EmbeddingResponse, + BatchChunkCreate, + SearchResponse, +) + +router = APIRouter(prefix="/documents", tags=["documents"]) + + +# Document Endpoints +@router.post("", response_model=DocumentResponse, status_code=status.HTTP_201_CREATED) +def create_document( + doc_data: DocumentCreate, db: Session = Depends(get_db) +): + """Create a new document.""" + document = DocumentService.create_document( + db=db, + title=doc_data.title, + source=doc_data.source, + metadata=doc_data.metadata, + ) + return document + + +@router.get("", response_model=List[DocumentResponse]) +def list_documents( + skip: int = 0, + limit: int = 100, + include_deleted: bool = False, + db: Session = Depends(get_db), +): + """List all documents with pagination.""" + documents = DocumentService.list_documents( + db=db, skip=skip, limit=limit, include_deleted=include_deleted + ) + return documents + + +@router.get("/{doc_id}", response_model=DocumentDetailResponse) +def get_document(doc_id: str, db: Session = Depends(get_db)): + """Retrieve a specific document with chunk count.""" + document = DocumentService.get_document(db=db, doc_id=doc_id) + if not document: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, detail="Document not found" + ) + + chunk_count = len(document.chunks) + return {**document.__dict__, "chunk_count": chunk_count} + + +@router.put("/{doc_id}", response_model=DocumentResponse) +def update_document( + doc_id: str, doc_data: DocumentUpdate, db: Session = Depends(get_db) +): + """Update a document.""" + document = DocumentService.update_document( + db=db, + doc_id=doc_id, + title=doc_data.title, + source=doc_data.source, + metadata=doc_data.metadata, + ) + if not document: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, detail="Document not found" + ) + return document + + +@router.delete("/{doc_id}", status_code=status.HTTP_204_NO_CONTENT) +def delete_document( + doc_id: str, hard_delete: bool = False, db: Session = Depends(get_db) +): + """Delete a document (soft or hard delete).""" + success = DocumentService.delete_document(db=db, doc_id=doc_id, hard_delete=hard_delete) + if not success: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, detail="Document not found" + ) + return None + + +@router.post("/search", response_model=SearchResponse) +def search_documents( + search_data: DocumentSearch, db: Session = Depends(get_db) +): + """Search documents by title or source.""" + results = DocumentService.search_documents(db=db, query_str=search_data.query) + return {"count": len(results), "results": [dict(d.__dict__) for d in results]} + + +# Chunk Endpoints +@router.post("/{doc_id}/chunks", response_model=ChunkResponse, status_code=status.HTTP_201_CREATED) +def create_chunk( + doc_id: str, chunk_data: ChunkCreate, db: Session = Depends(get_db) +): + """Create a chunk for a document.""" + document = DocumentService.get_document(db=db, doc_id=doc_id) + if not document: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, detail="Document not found" + ) + + chunk = ChunkService.create_chunk( + db=db, + document_id=doc_id, + chunk_index=chunk_data.chunk_index, + text=chunk_data.text, + metadata=chunk_data.metadata, + ) + return chunk + + +@router.post("/{doc_id}/chunks/batch", response_model=List[ChunkResponse], status_code=status.HTTP_201_CREATED) +def batch_create_chunks( + doc_id: str, batch_data: BatchChunkCreate, db: Session = Depends(get_db) +): + """Batch create chunks for a document.""" + document = DocumentService.get_document(db=db, doc_id=doc_id) + if not document: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, detail="Document not found" + ) + + chunks = [] + for chunk_data in batch_data.chunks: + chunk = ChunkService.create_chunk( + db=db, + document_id=doc_id, + chunk_index=chunk_data.chunk_index, + text=chunk_data.text, + metadata=chunk_data.metadata, + ) + chunks.append(chunk) + + return chunks + + +@router.get("/{doc_id}/chunks", response_model=List[ChunkResponse]) +def list_document_chunks( + doc_id: str, + skip: int = 0, + limit: int = 1000, + db: Session = Depends(get_db), +): + """List all chunks for a document.""" + document = DocumentService.get_document(db=db, doc_id=doc_id) + if not document: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, detail="Document not found" + ) + + chunks = ChunkService.list_chunks_by_document( + db=db, document_id=doc_id, skip=skip, limit=limit + ) + return chunks + + +@router.get("/chunks/{chunk_id}", response_model=ChunkDetailResponse) +def get_chunk(chunk_id: str, db: Session = Depends(get_db)): + """Retrieve a specific chunk.""" + chunk = ChunkService.get_chunk(db=db, chunk_id=chunk_id) + if not chunk: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, detail="Chunk not found" + ) + + embedding_count = len(chunk.embeddings) + has_embedding = embedding_count > 0 + + return {**chunk.__dict__, "embedding_count": embedding_count, "has_embedding": has_embedding} + + +@router.put("/chunks/{chunk_id}", response_model=ChunkResponse) +def update_chunk( + chunk_id: str, chunk_data: ChunkUpdate, db: Session = Depends(get_db) +): + """Update a chunk.""" + chunk = ChunkService.update_chunk( + db=db, + chunk_id=chunk_id, + text=chunk_data.text, + metadata=chunk_data.metadata, + ) + if not chunk: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, detail="Chunk not found" + ) + return chunk + + +@router.delete("/chunks/{chunk_id}", status_code=status.HTTP_204_NO_CONTENT) +def delete_chunk( + chunk_id: str, hard_delete: bool = False, db: Session = Depends(get_db) +): + """Delete a chunk.""" + success = ChunkService.delete_chunk(db=db, chunk_id=chunk_id, hard_delete=hard_delete) + if not success: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, detail="Chunk not found" + ) + return None + + +# Embedding Endpoints +@router.post("/chunks/{chunk_id}/embeddings", response_model=EmbeddingResponse, status_code=status.HTTP_201_CREATED) +def create_embedding( + chunk_id: str, embedding_data: EmbeddingCreate, db: Session = Depends(get_db) +): + """Create an embedding for a chunk.""" + chunk = ChunkService.get_chunk(db=db, chunk_id=chunk_id) + if not chunk: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, detail="Chunk not found" + ) + + embedding = EmbeddingService.create_embedding( + db=db, + chunk_id=chunk_id, + vector=embedding_data.vector, + model=embedding_data.model, + ) + return embedding + + +@router.get("/embeddings/{embedding_id}", response_model=EmbeddingResponse) +def get_embedding(embedding_id: str, db: Session = Depends(get_db)): + """Retrieve a specific embedding.""" + embedding = EmbeddingService.get_embedding(db=db, embedding_id=embedding_id) + if not embedding: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, detail="Embedding not found" + ) + return embedding + + +@router.put("/embeddings/{embedding_id}", response_model=EmbeddingResponse) +def update_embedding( + embedding_id: str, + embedding_data: EmbeddingUpdate, + db: Session = Depends(get_db), +): + """Update an embedding.""" + embedding = EmbeddingService.update_embedding( + db=db, + embedding_id=embedding_id, + vector=embedding_data.vector, + chroma_id=embedding_data.chroma_id, + ) + if not embedding: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, detail="Embedding not found" + ) + return embedding + + +@router.delete("/embeddings/{embedding_id}", status_code=status.HTTP_204_NO_CONTENT) +def delete_embedding(embedding_id: str, db: Session = Depends(get_db)): + """Delete an embedding.""" + success = EmbeddingService.delete_embedding(db=db, embedding_id=embedding_id) + if not success: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, detail="Embedding not found" + ) + return None diff --git a/backend/app/api/endpoints/health.py b/backend/app/api/endpoints/health.py new file mode 100644 index 0000000..7e588b5 --- /dev/null +++ b/backend/app/api/endpoints/health.py @@ -0,0 +1,18 @@ +from fastapi import APIRouter +from fastapi.responses import JSONResponse + +router = APIRouter() + +# Both /api/health and /api/health/ answer directly: a redirect between them +# breaks browser fetches behind HTTPS proxies (Cloud Run redirects to http://). +@router.get("", response_model=dict) +@router.get("/", response_model=dict, include_in_schema=False) +async def health_check(): + """ + Health check endpoint to verify backend is running. + Returns a simple success response with HTTP 200 status. + """ + return JSONResponse( + status_code=200, + content={"status": "healthy"} + ) \ No newline at end of file diff --git a/backend/app/api/endpoints/query.py b/backend/app/api/endpoints/query.py new file mode 100644 index 0000000..5b7e573 --- /dev/null +++ b/backend/app/api/endpoints/query.py @@ -0,0 +1,14 @@ +from fastapi import APIRouter, Depends +from app.services.rag_service import RAGService +from app.models.requests import QueryRequest +from app.models.response import QueryResponse + + +router = APIRouter() + +@router.post("/", response_model=QueryResponse) +async def query(request: QueryRequest): + rag = RAGService() + answer = await rag.answer(request.query) + return QueryResponse(answer=answer) + diff --git a/backend/app/api/endpoints/rag_endpoint.py b/backend/app/api/endpoints/rag_endpoint.py new file mode 100644 index 0000000..547f326 --- /dev/null +++ b/backend/app/api/endpoints/rag_endpoint.py @@ -0,0 +1,71 @@ +import json +import logging + +from fastapi import APIRouter, Depends +from fastapi.responses import StreamingResponse +from pydantic import BaseModel + +from app.services.rag_chain_service import get_rag_chain_service + +logger = logging.getLogger(__name__) + +router = APIRouter() + + +class ChatRequest(BaseModel): + message: str + + +def get_chain(): + """Dependency so tests can swap in a fake chain.""" + return get_rag_chain_service().get_chain() + + +def _sse(event: dict) -> str: + return f"data: {json.dumps(event)}\n\n" + + +async def stream_rag_response(chain, message: str): + """ + SSE events, in order: + {"type": "source", "message": "Source n: ", "index", "url", "title"?} per doc + {"type": "text", "message": } answer chunks, to be appended + {"type": "done"} + """ + try: + sources_sent = False + answer_chars = 0 + # astream keeps retrieval and generation off the event loop and yields the + # retrieved context as soon as it is ready, then answer tokens as produced. + async for chunk in chain.astream(message): + if "context" in chunk and not sources_sent: + sources_sent = True + for i, doc in enumerate(chunk["context"], start=1): + url = doc.metadata.get("source", "Unknown") + event = { + "type": "source", + "message": f"Source {i}: {url}", + "index": i, + "url": url, + } + if doc.metadata.get("title"): + event["title"] = doc.metadata["title"] + yield _sse(event) + delta = chunk.get("answer") + if delta: + answer_chars += len(delta) + yield _sse({"type": "text", "message": delta}) + logger.info("Streamed answer: %d characters", answer_chars) + except Exception: + logger.exception("Error in RAG streaming") + yield _sse({"type": "text", "message": "Error processing request"}) + yield _sse({"type": "done"}) + + +@router.post("/chat") +async def chat(request: ChatRequest, chain=Depends(get_chain)): + return StreamingResponse( + stream_rag_response(chain, request.message), + media_type="text/event-stream", + headers={"Cache-Control": "no-cache", "X-Accel-Buffering": "no"}, + ) diff --git a/backend/app/core/__init__.py b/backend/app/core/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/backend/app/core/config.py b/backend/app/core/config.py new file mode 100644 index 0000000..402c2e8 --- /dev/null +++ b/backend/app/core/config.py @@ -0,0 +1,94 @@ +""" +Application settings, read from the environment and backend/.env. + +Relative filesystem paths (CHROMA_PERSIST_DIRECTORY, a relative SQLite +DATABASE_URL) resolve against the backend/ directory, so the API and the scripts +see the same files whatever directory they are started from. +""" + +from functools import lru_cache +from pathlib import Path +from typing import Annotated, Literal, Optional + +from pydantic import field_validator +from pydantic_settings import BaseSettings, NoDecode, SettingsConfigDict + +BACKEND_DIR = Path(__file__).resolve().parents[2] +REPO_ROOT = BACKEND_DIR.parent + + +def _resolve(path: str) -> str: + p = Path(path) + return str(p if p.is_absolute() else (BACKEND_DIR / p).resolve()) + + +class Settings(BaseSettings): + model_config = SettingsConfigDict( + env_file=BACKEND_DIR / ".env", + env_file_encoding="utf-8", + extra="ignore", + # Validation errors must never echo secrets (API keys, JWT secret) to logs. + hide_input_in_errors=True, + ) + + environment: str = "development" + + # Database + database_url: str = "sqlite:///./asktemoc.db" + db_echo: bool = False + + # Vector store + chroma_persist_directory: str = "./app/chroma_db" + chroma_collection_name: str = "asktemoc_collection" + catalog_data_dir: str = str(REPO_ROOT / "data" / "catalog") + + # LLM / embeddings: OpenAI when OPENAI_API_KEY is set, otherwise Ollama. + openai_api_key: Optional[str] = None + openai_model: str = "gpt-4o-mini" + openai_embedding_model: str = "text-embedding-3-small" + ollama_base_url: str = "http://localhost:11434" + ollama_model: str = "llama3.1:8b" + ollama_embedding_model: str = "nomic-embed-text" + + # Comma-separated in the environment. + cors_origins: Annotated[list[str], NoDecode] = ["http://localhost:3000"] + + @field_validator("cors_origins", mode="before") + @classmethod + def _split_origins(cls, v): + if isinstance(v, str): + return [o.strip() for o in v.split(",") if o.strip()] + return v + + @field_validator("openai_api_key", mode="before") + @classmethod + def _blank_is_none(cls, v): + return v or None + + @field_validator("chroma_persist_directory", "catalog_data_dir") + @classmethod + def _absolute_dir(cls, v: str) -> str: + return _resolve(v) + + @field_validator("database_url") + @classmethod + def _absolute_sqlite(cls, v: str) -> str: + prefix = "sqlite:///" + if v.startswith(prefix) and v != "sqlite:///:memory:": + return prefix + Path(_resolve(v[len(prefix):])).as_posix() + return v + + @property + def llm_provider(self) -> Literal["openai", "ollama"]: + return "openai" if self.openai_api_key else "ollama" + + @property + def embedding_model(self) -> str: + if self.llm_provider == "openai": + return self.openai_embedding_model + return self.ollama_embedding_model + + +@lru_cache +def get_settings() -> Settings: + return Settings() diff --git a/backend/app/db/__init__.py b/backend/app/db/__init__.py new file mode 100644 index 0000000..bdd0627 --- /dev/null +++ b/backend/app/db/__init__.py @@ -0,0 +1,22 @@ +""" +Database module for AskTemoc backend. +""" + +from app.db.database import get_db, init_db, drop_db, engine, SessionLocal +from app.db.models import Base, Document, Chunk, Embedding +from app.db.services import DocumentService, ChunkService, EmbeddingService + +__all__ = [ + "get_db", + "init_db", + "drop_db", + "engine", + "SessionLocal", + "Base", + "Document", + "Chunk", + "Embedding", + "DocumentService", + "ChunkService", + "EmbeddingService", +] diff --git a/backend/app/db/database.py b/backend/app/db/database.py new file mode 100644 index 0000000..892dcda --- /dev/null +++ b/backend/app/db/database.py @@ -0,0 +1,48 @@ +""" +Database session management and initialization. +""" + +from sqlalchemy import create_engine +from sqlalchemy.orm import sessionmaker, Session +from app.core.config import get_settings +from app.db.models import Base + +# Database URL configuration +DATABASE_URL = get_settings().database_url + +# Create engine (connects lazily, on first use) +engine = create_engine( + DATABASE_URL, + connect_args={"check_same_thread": False} if DATABASE_URL.startswith("sqlite") else {}, + echo=get_settings().db_echo, +) + +# Create session factory +SessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine) + + +def get_db() -> Session: + """ + Dependency injection for FastAPI to get a database session. + """ + db = SessionLocal() + try: + yield db + finally: + db.close() + + +def init_db(): + """ + Initialize database by creating all tables. + """ + Base.metadata.create_all(bind=engine) + print("Database tables created successfully.") + + +def drop_db(): + """ + Drop all tables (use with caution). + """ + Base.metadata.drop_all(bind=engine) + print("All database tables dropped.") diff --git a/backend/app/db/models.py b/backend/app/db/models.py new file mode 100644 index 0000000..e46ffa8 --- /dev/null +++ b/backend/app/db/models.py @@ -0,0 +1,87 @@ +""" +Database models for documents, chunks, and embeddings. +""" + +from datetime import datetime +from sqlalchemy import Column, String, Integer, Float, DateTime, ForeignKey, Text, JSON, Boolean, Index +from sqlalchemy.ext.declarative import declarative_base +from sqlalchemy.orm import relationship + +Base = declarative_base() + + +class Document(Base): + """ + Stores document metadata with optional embedding information. + """ + __tablename__ = "documents" + + id = Column(String, primary_key=True, index=True) # UUID or custom ID + title = Column(String(255), nullable=False, index=True) + source = Column(String(512), nullable=True) # URL, file path, etc. + doc_metadata = Column(JSON, nullable=True) # Flexible metadata storage + created_at = Column(DateTime, default=datetime.utcnow, index=True) + updated_at = Column(DateTime, default=datetime.utcnow, onupdate=datetime.utcnow) + is_deleted = Column(Boolean, default=False, index=True) # Soft delete + + # Relationships + chunks = relationship("Chunk", back_populates="document", cascade="all, delete-orphan") + + def __repr__(self): + return f"" + + +class Chunk(Base): + """ + Stores text chunks extracted from documents. + """ + __tablename__ = "chunks" + + id = Column(String, primary_key=True, index=True) # UUID or chunk_id + document_id = Column(String, ForeignKey("documents.id"), nullable=False, index=True) + chunk_index = Column(Integer, nullable=False) # Sequence position within document + text = Column(Text, nullable=False) + chunk_metadata = Column(JSON, nullable=True) # Custom metadata for chunk + created_at = Column(DateTime, default=datetime.utcnow, index=True) + updated_at = Column(DateTime, default=datetime.utcnow, onupdate=datetime.utcnow) + is_deleted = Column(Boolean, default=False, index=True) # Soft delete + + # Relationships + document = relationship("Document", back_populates="chunks") + embeddings = relationship("Embedding", back_populates="chunk", cascade="all, delete-orphan") + + __table_args__ = ( + Index("idx_document_chunk_index", "document_id", "chunk_index"), + ) + + def __repr__(self): + return f"" + + +class Embedding(Base): + """ + Stores embedding vectors and related metadata. + Links chunks to their vector representations. + """ + __tablename__ = "embeddings" + + id = Column(String, primary_key=True, index=True) # UUID or chroma_id + chunk_id = Column(String, ForeignKey("chunks.id"), nullable=False, index=True) + vector = Column(JSON, nullable=True) # Store as JSON array for flexibility + model = Column(String(100), nullable=True) # Model used (e.g., "text-embedding-ada-002") + chroma_id = Column(String(255), nullable=True, index=True) # Reference to ChromaDB ID + is_synced = Column(Boolean, default=False, index=True) # Track sync status + created_at = Column(DateTime, default=datetime.utcnow, index=True) + updated_at = Column(DateTime, default=datetime.utcnow, onupdate=datetime.utcnow) + last_synced_at = Column(DateTime, nullable=True) + + # Relationships + chunk = relationship("Chunk", back_populates="embeddings") + + __table_args__ = ( + Index("idx_chroma_id", "chroma_id"), + Index("idx_is_synced", "is_synced"), + ) + + def __repr__(self): + return f"" diff --git a/backend/app/db/services.py b/backend/app/db/services.py new file mode 100644 index 0000000..91c05f7 --- /dev/null +++ b/backend/app/db/services.py @@ -0,0 +1,336 @@ +""" +Database service layer for CRUD operations on documents, chunks, and embeddings. +""" + +from typing import List, Optional, Dict, Any +from datetime import datetime +from sqlalchemy.orm import Session +from sqlalchemy import and_, or_ +from app.db.models import Document, Chunk, Embedding +import uuid + + +class DocumentService: + """Service for document CRUD operations.""" + + @staticmethod + def create_document( + db: Session, + title: str, + source: Optional[str] = None, + metadata: Optional[Dict[str, Any]] = None, + doc_id: Optional[str] = None, + ) -> Document: + """Create a new document.""" + document = Document( + id=doc_id or str(uuid.uuid4()), + title=title, + source=source, + doc_metadata=metadata or {}, + ) + db.add(document) + db.commit() + db.refresh(document) + return document + + @staticmethod + def get_document(db: Session, doc_id: str) -> Optional[Document]: + """Retrieve a document by ID.""" + return db.query(Document).filter( + and_(Document.id == doc_id, Document.is_deleted == False) + ).first() + + @staticmethod + def list_documents( + db: Session, skip: int = 0, limit: int = 100, include_deleted: bool = False + ) -> List[Document]: + """List all documents with pagination.""" + query = db.query(Document) + if not include_deleted: + query = query.filter(Document.is_deleted == False) + return query.offset(skip).limit(limit).all() + + @staticmethod + def update_document( + db: Session, + doc_id: str, + title: Optional[str] = None, + source: Optional[str] = None, + metadata: Optional[Dict[str, Any]] = None, + ) -> Optional[Document]: + """Update a document.""" + document = DocumentService.get_document(db, doc_id) + if not document: + return None + + if title is not None: + document.title = title + if source is not None: + document.source = source + if metadata is not None: + document.doc_metadata = {**(document.doc_metadata or {}), **metadata} + + document.updated_at = datetime.utcnow() + db.commit() + db.refresh(document) + return document + + @staticmethod + def delete_document(db: Session, doc_id: str, hard_delete: bool = False) -> bool: + """Soft or hard delete a document.""" + document = db.query(Document).filter(Document.id == doc_id).first() + if not document: + return False + + if hard_delete: + # Hard delete document and all related chunks/embeddings + db.delete(document) + else: + # Soft delete + document.is_deleted = True + document.updated_at = datetime.utcnow() + + db.commit() + return True + + @staticmethod + def search_documents(db: Session, query_str: str) -> List[Document]: + """Search documents by title or source.""" + return db.query(Document).filter( + and_( + Document.is_deleted == False, + or_( + Document.title.ilike(f"%{query_str}%"), + Document.source.ilike(f"%{query_str}%"), + ), + ) + ).all() + + +class ChunkService: + """Service for chunk CRUD operations.""" + + @staticmethod + def create_chunk( + db: Session, + document_id: str, + chunk_index: int, + text: str, + metadata: Optional[Dict[str, Any]] = None, + chunk_id: Optional[str] = None, + ) -> Chunk: + """Create a new chunk.""" + chunk = Chunk( + id=chunk_id or str(uuid.uuid4()), + document_id=document_id, + chunk_index=chunk_index, + text=text, + chunk_metadata=metadata or {}, + ) + db.add(chunk) + db.commit() + db.refresh(chunk) + return chunk + + @staticmethod + def get_chunk(db: Session, chunk_id: str) -> Optional[Chunk]: + """Retrieve a chunk by ID.""" + return db.query(Chunk).filter( + and_(Chunk.id == chunk_id, Chunk.is_deleted == False) + ).first() + + @staticmethod + def list_chunks_by_document( + db: Session, document_id: str, skip: int = 0, limit: int = 1000 + ) -> List[Chunk]: + """List all chunks for a document.""" + return ( + db.query(Chunk) + .filter( + and_( + Chunk.document_id == document_id, + Chunk.is_deleted == False, + ) + ) + .order_by(Chunk.chunk_index) + .offset(skip) + .limit(limit) + .all() + ) + + @staticmethod + def update_chunk( + db: Session, + chunk_id: str, + text: Optional[str] = None, + metadata: Optional[Dict[str, Any]] = None, + ) -> Optional[Chunk]: + """Update a chunk.""" + chunk = ChunkService.get_chunk(db, chunk_id) + if not chunk: + return None + + if text is not None: + chunk.text = text + if metadata is not None: + chunk.chunk_metadata = {**(chunk.chunk_metadata or {}), **metadata} + + chunk.updated_at = datetime.utcnow() + db.commit() + db.refresh(chunk) + return chunk + + @staticmethod + def delete_chunk(db: Session, chunk_id: str, hard_delete: bool = False) -> bool: + """Soft or hard delete a chunk.""" + chunk = db.query(Chunk).filter(Chunk.id == chunk_id).first() + if not chunk: + return False + + if hard_delete: + db.delete(chunk) + else: + chunk.is_deleted = True + chunk.updated_at = datetime.utcnow() + + db.commit() + return True + + @staticmethod + def get_chunks_by_ids(db: Session, chunk_ids: List[str]) -> List[Chunk]: + """Retrieve multiple chunks by IDs.""" + return ( + db.query(Chunk) + .filter( + and_( + Chunk.id.in_(chunk_ids), + Chunk.is_deleted == False, + ) + ) + .all() + ) + + +class EmbeddingService: + """Service for embedding CRUD operations.""" + + @staticmethod + def create_embedding( + db: Session, + chunk_id: str, + vector: List[float], + model: str = "text-embedding-ada-002", + chroma_id: Optional[str] = None, + embedding_id: Optional[str] = None, + ) -> Embedding: + """Create a new embedding.""" + embedding = Embedding( + id=embedding_id or str(uuid.uuid4()), + chunk_id=chunk_id, + vector=vector, + model=model, + chroma_id=chroma_id, + is_synced=False, + ) + db.add(embedding) + db.commit() + db.refresh(embedding) + return embedding + + @staticmethod + def get_embedding(db: Session, embedding_id: str) -> Optional[Embedding]: + """Retrieve an embedding by ID.""" + return db.query(Embedding).filter(Embedding.id == embedding_id).first() + + @staticmethod + def get_embedding_by_chunk(db: Session, chunk_id: str) -> Optional[Embedding]: + """Retrieve an embedding by chunk ID.""" + return db.query(Embedding).filter(Embedding.chunk_id == chunk_id).first() + + @staticmethod + def list_embeddings_by_chunk(db: Session, chunk_id: str) -> List[Embedding]: + """List all embeddings for a chunk.""" + return db.query(Embedding).filter(Embedding.chunk_id == chunk_id).all() + + @staticmethod + def list_unsynced_embeddings(db: Session, limit: int = 100) -> List[Embedding]: + """List embeddings not yet synced to ChromaDB.""" + return ( + db.query(Embedding) + .filter(Embedding.is_synced == False) + .limit(limit) + .all() + ) + + @staticmethod + def update_embedding( + db: Session, + embedding_id: str, + vector: Optional[List[float]] = None, + chroma_id: Optional[str] = None, + is_synced: Optional[bool] = None, + ) -> Optional[Embedding]: + """Update an embedding.""" + embedding = EmbeddingService.get_embedding(db, embedding_id) + if not embedding: + return None + + if vector is not None: + embedding.vector = vector + if chroma_id is not None: + embedding.chroma_id = chroma_id + if is_synced is not None: + embedding.is_synced = is_synced + if is_synced: + embedding.last_synced_at = datetime.utcnow() + + embedding.updated_at = datetime.utcnow() + db.commit() + db.refresh(embedding) + return embedding + + @staticmethod + def delete_embedding(db: Session, embedding_id: str) -> bool: + """Delete an embedding.""" + embedding = db.query(Embedding).filter(Embedding.id == embedding_id).first() + if not embedding: + return False + + db.delete(embedding) + db.commit() + return True + + @staticmethod + def mark_synced( + db: Session, embedding_id: str, chroma_id: str + ) -> Optional[Embedding]: + """Mark an embedding as synced to ChromaDB.""" + embedding = EmbeddingService.get_embedding(db, embedding_id) + if not embedding: + return None + + embedding.is_synced = True + embedding.chroma_id = chroma_id + embedding.last_synced_at = datetime.utcnow() + db.commit() + db.refresh(embedding) + return embedding + + @staticmethod + def get_embeddings_by_document(db: Session, document_id: str) -> List[Embedding]: + """Retrieve all embeddings for a document.""" + return ( + db.query(Embedding) + .join(Chunk) + .filter(Chunk.document_id == document_id) + .all() + ) + + @staticmethod + def get_embeddings_by_ids(db: Session, embedding_ids: List[str]) -> List[Embedding]: + """Retrieve multiple embeddings by IDs.""" + return ( + db.query(Embedding) + .filter(Embedding.id.in_(embedding_ids)) + .all() + ) diff --git a/backend/app/main.py b/backend/app/main.py new file mode 100644 index 0000000..3943a2a --- /dev/null +++ b/backend/app/main.py @@ -0,0 +1,35 @@ +from fastapi import FastAPI +from fastapi.middleware.cors import CORSMiddleware +import logging + +from app.api.endpoints import query, documents, chroma, dashboard, rag_endpoint, health +from app.core.config import get_settings +from app.db.database import init_db + +logging.basicConfig(level=logging.INFO) + +settings = get_settings() + +app = FastAPI(title="AskTemoc Backend") + +# Add CORS middleware +app.add_middleware( + CORSMiddleware, + allow_origins=settings.cors_origins, + allow_credentials=True, + allow_methods=["*"], + allow_headers=["*"], +) + +# Initialize database on startup +@app.on_event("startup") +def startup_event(): + init_db() + +# Include routers +app.include_router(query.router, prefix="/api/query", tags=['query']) +app.include_router(documents.router, prefix="/api", tags=['documents']) +app.include_router(chroma.router, prefix="/api", tags=['chroma']) +app.include_router(dashboard.router, prefix="/api", tags=['dashboard']) +app.include_router(rag_endpoint.router, prefix="/api", tags=['rag']) +app.include_router(health.router, prefix="/api/health", tags=['health']) \ No newline at end of file diff --git a/backend/app/models/__init__.py b/backend/app/models/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/backend/app/models/requests.py b/backend/app/models/requests.py new file mode 100644 index 0000000..5b975f9 --- /dev/null +++ b/backend/app/models/requests.py @@ -0,0 +1,4 @@ +from pydantic import BaseModel + +class QueryRequest(BaseModel): + query: str \ No newline at end of file diff --git a/backend/app/models/response.py b/backend/app/models/response.py new file mode 100644 index 0000000..2e06d91 --- /dev/null +++ b/backend/app/models/response.py @@ -0,0 +1,6 @@ +from pydantic import BaseModel +from typing import List, Dict, Union + +class QueryResponse(BaseModel): + answer: str + # sources: List[Union[str, Dict]] \ No newline at end of file diff --git a/backend/app/schemas/__init__.py b/backend/app/schemas/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/backend/app/schemas/db_schemas.py b/backend/app/schemas/db_schemas.py new file mode 100644 index 0000000..99085da --- /dev/null +++ b/backend/app/schemas/db_schemas.py @@ -0,0 +1,149 @@ +""" +Pydantic schemas for API requests and responses. +""" + +from typing import Optional, List, Dict, Any +from datetime import datetime +from pydantic import BaseModel, Field + + +# Document Schemas +class DocumentCreate(BaseModel): + """Schema for creating a document.""" + title: str = Field(..., min_length=1, max_length=255) + source: Optional[str] = Field(None, max_length=512) + metadata: Optional[Dict[str, Any]] = None + + +class DocumentUpdate(BaseModel): + """Schema for updating a document.""" + title: Optional[str] = Field(None, min_length=1, max_length=255) + source: Optional[str] = Field(None, max_length=512) + metadata: Optional[Dict[str, Any]] = None + + +class DocumentResponse(BaseModel): + """Schema for document response.""" + id: str + title: str + source: Optional[str] + metadata: Optional[Dict[str, Any]] + created_at: datetime + updated_at: datetime + is_deleted: bool + + class Config: + from_attributes = True + + +class DocumentDetailResponse(DocumentResponse): + """Detailed document response with chunk count.""" + chunk_count: int = 0 + + +# Chunk Schemas +class ChunkCreate(BaseModel): + """Schema for creating a chunk.""" + document_id: str + chunk_index: int = Field(..., ge=0) + text: str = Field(..., min_length=1) + metadata: Optional[Dict[str, Any]] = None + + +class ChunkUpdate(BaseModel): + """Schema for updating a chunk.""" + text: Optional[str] = Field(None, min_length=1) + metadata: Optional[Dict[str, Any]] = None + + +class ChunkResponse(BaseModel): + """Schema for chunk response.""" + id: str + document_id: str + chunk_index: int + text: str + metadata: Optional[Dict[str, Any]] + created_at: datetime + updated_at: datetime + is_deleted: bool + + class Config: + from_attributes = True + + +class ChunkDetailResponse(ChunkResponse): + """Detailed chunk response with embedding info.""" + embedding_count: int = 0 + has_embedding: bool = False + + +# Embedding Schemas +class EmbeddingCreate(BaseModel): + """Schema for creating an embedding.""" + chunk_id: str + vector: List[float] = Field(..., min_items=1) + model: Optional[str] = Field("text-embedding-ada-002", max_length=100) + + +class EmbeddingUpdate(BaseModel): + """Schema for updating an embedding.""" + vector: Optional[List[float]] = None + chroma_id: Optional[str] = None + + +class EmbeddingResponse(BaseModel): + """Schema for embedding response.""" + id: str + chunk_id: str + model: Optional[str] + chroma_id: Optional[str] + is_synced: bool + created_at: datetime + updated_at: datetime + last_synced_at: Optional[datetime] + + class Config: + from_attributes = True + + +# Batch Operations +class BatchChunkCreate(BaseModel): + """Schema for batch creating chunks.""" + document_id: str + chunks: List[ChunkCreate] + + +class BatchEmbeddingSync(BaseModel): + """Schema for batch syncing embeddings to ChromaDB.""" + embedding_ids: Optional[List[str]] = None + document_id: Optional[str] = None + sync_all_unsynced: bool = False + + +# ChromaDB Export Responses +class ChromaExportResponse(BaseModel): + """Response for ChromaDB export operation.""" + status: str + message: Optional[str] = None + upserted_count: Optional[int] = None + updated_db_count: Optional[int] = None + error: Optional[str] = None + + +class ChromaIndexStats(BaseModel): + """ChromaDB index statistics.""" + status: str + stats: Optional[Dict[str, Any]] = None + error: Optional[str] = None + + +# Search and Query +class DocumentSearch(BaseModel): + """Schema for searching documents.""" + query: str = Field(..., min_length=1) + + +class SearchResponse(BaseModel): + """Generic search response.""" + count: int + results: List[Dict[str, Any]] diff --git a/backend/app/services/__init__.py b/backend/app/services/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/backend/app/services/chroma_service.py b/backend/app/services/chroma_service.py new file mode 100644 index 0000000..821bf01 --- /dev/null +++ b/backend/app/services/chroma_service.py @@ -0,0 +1,266 @@ +""" +Chroma export pipeline for syncing embeddings and metadata. +""" + +import os +from typing import List, Dict, Any, Optional +from sqlalchemy.orm import Session +from app.core.config import get_settings +from app.db.models import Embedding, Chunk, Document +from app.db.services import EmbeddingService + +try: + import chromadb + from chromadb.api.models.Collection import Collection +except ImportError: + chromadb = None + Collection = None + + +class ChromaService: + """Service for exporting embeddings and metadata to ChromaDB.""" + + def __init__(self): + """Initialize ChromaDB client.""" + settings = get_settings() + self.persist_directory = settings.chroma_persist_directory + self.collection_name = settings.chroma_collection_name + self.client = None + self.collection = None + + if chromadb is None: + raise ImportError("chromadb package not installed") + + self._initialize_client() + + def _initialize_client(self): + """Initialize ChromaDB client and collection.""" + # Create persist directory if it doesn't exist + os.makedirs(self.persist_directory, exist_ok=True) + + # Initialize persistent client + self.client = chromadb.PersistentClient(path=self.persist_directory) + + # Get or create collection + try: + self.collection = self.client.get_collection(name=self.collection_name) + except Exception: + # Collection doesn't exist, create it + self.collection = self.client.create_collection(name=self.collection_name) + print(f"Created ChromaDB collection: {self.collection_name}") + + def prepare_vectors_for_upsert( + self, db: Session, embeddings: List[Embedding] + ) -> Dict[str, Any]: + """ + Prepare vectors and metadata for Chroma upsert. + + Returns dict with 'ids', 'embeddings', 'metadatas', 'documents'. + """ + ids = [] + vectors = [] + metadatas = [] + documents = [] + + for embedding in embeddings: + chunk = db.query(Chunk).filter(Chunk.id == embedding.chunk_id).first() + if not chunk: + continue + + document = db.query(Document).filter( + Document.id == chunk.document_id + ).first() + if not document: + continue + + # Prepare metadata + metadata = { + "embedding_id": embedding.id, + "chunk_id": chunk.id, + "document_id": document.id, + "document_title": document.title, + "document_source": document.source or "", + "chunk_index": chunk.chunk_index, + "created_at": chunk.created_at.isoformat() if chunk.created_at else "", + } + + # Add custom metadata from chunk + if chunk.chunk_metadata: + metadata.update(chunk.chunk_metadata) + + # Add custom metadata from document + if document.doc_metadata: + metadata.update(document.doc_metadata) + + # Use embedding ID as vector ID (or chroma_id if already set) + vector_id = embedding.chroma_id or embedding.id + + ids.append(vector_id) + vectors.append(embedding.vector) + metadatas.append(metadata) + documents.append(chunk.text) + + return { + "ids": ids, + "embeddings": vectors, + "metadatas": metadatas, + "documents": documents, + } + + def upsert_vectors(self, db: Session, embeddings: List[Embedding]) -> Dict[str, Any]: + """ + Upsert embeddings and metadata to ChromaDB. + + Returns a dictionary with upsert results and statistics. + """ + if not self.collection: + raise RuntimeError("ChromaDB collection not initialized") + + data = self.prepare_vectors_for_upsert(db, embeddings) + + if not data["ids"]: + return {"status": "no_vectors", "count": 0} + + try: + # Upsert vectors to ChromaDB + # Chroma's add method will overwrite if IDs already exist + self.collection.add( + ids=data["ids"], + embeddings=data["embeddings"], + metadatas=data["metadatas"], + documents=data["documents"], + ) + + # Update sync status in database + updated_ids = [] + for vector_id in data["ids"]: + # Find embedding by ID or chroma_id + embedding = db.query(Embedding).filter( + (Embedding.id == vector_id) | (Embedding.chroma_id == vector_id) + ).first() + if embedding: + EmbeddingService.mark_synced(db, embedding.id, vector_id) + updated_ids.append(embedding.id) + + return { + "status": "success", + "upserted_count": len(data["ids"]), + "updated_db_count": len(updated_ids), + } + + except Exception as e: + return { + "status": "error", + "error": str(e), + "attempted_count": len(data["ids"]), + } + + def export_document_embeddings( + self, db: Session, document_id: str + ) -> Dict[str, Any]: + """Export all embeddings for a specific document to ChromaDB.""" + embeddings = EmbeddingService.get_embeddings_by_document(db, document_id) + unsynced = [e for e in embeddings if not e.is_synced] + + if not unsynced: + return { + "status": "no_new_embeddings", + "total_embeddings": len(embeddings), + "synced_embeddings": len([e for e in embeddings if e.is_synced]), + } + + return self.upsert_vectors(db, unsynced) + + def export_unsynced_embeddings( + self, db: Session, batch_size: int = 100 + ) -> Dict[str, Any]: + """Export all unsynced embeddings to ChromaDB.""" + unsynced = EmbeddingService.list_unsynced_embeddings(db, limit=batch_size) + + if not unsynced: + return { + "status": "no_embeddings", + "count": 0, + } + + return self.upsert_vectors(db, unsynced) + + def delete_from_chroma( + self, vector_ids: List[str] + ) -> Dict[str, Any]: + """Delete vectors from ChromaDB by ID.""" + if not self.collection: + raise RuntimeError("ChromaDB collection not initialized") + + try: + self.collection.delete(ids=vector_ids) + return { + "status": "success", + "deleted_count": len(vector_ids), + } + except Exception as e: + return { + "status": "error", + "error": str(e), + "attempted_count": len(vector_ids), + } + + def search_chroma( + self, + query_vector: List[float], + top_k: int = 10, + where: Optional[Dict[str, Any]] = None, + ) -> Dict[str, Any]: + """Search ChromaDB collection.""" + if not self.collection: + raise RuntimeError("ChromaDB collection not initialized") + + try: + results = self.collection.query( + query_embeddings=[query_vector], + n_results=top_k, + where=where, + ) + + # Format results to be similar to Pinecone's format + matches = [] + if results['ids'] and len(results['ids'][0]) > 0: + for i, vec_id in enumerate(results['ids'][0]): + match = { + "id": vec_id, + "score": results['distances'][0][i] if 'distances' in results else None, + "metadata": results['metadatas'][0][i] if 'metadatas' in results else {}, + "document": results['documents'][0][i] if 'documents' in results else "", + } + matches.append(match) + + return { + "status": "success", + "matches": matches, + } + except Exception as e: + return { + "status": "error", + "error": str(e), + } + + def get_collection_stats(self) -> Dict[str, Any]: + """Get statistics about the ChromaDB collection.""" + if not self.collection: + raise RuntimeError("ChromaDB collection not initialized") + + try: + count = self.collection.count() + return { + "status": "success", + "stats": { + "count": count, + "name": self.collection_name, + "persist_directory": self.persist_directory, + }, + } + except Exception as e: + return { + "status": "error", + "error": str(e), + } diff --git a/backend/app/services/data_ingestion_service.py b/backend/app/services/data_ingestion_service.py new file mode 100644 index 0000000..bd70952 --- /dev/null +++ b/backend/app/services/data_ingestion_service.py @@ -0,0 +1,171 @@ +""" +Data ingestion service for loading university program data into ChromaDB. +This service processes all program requirement files and stores them as embeddings. +""" + +import os +import logging +from typing import List, Dict, Any +from langchain_core.documents import Document +from langchain_text_splitters import RecursiveCharacterTextSplitter +import chromadb +from langchain_chroma import Chroma + +from app.core.config import get_settings +from app.services.retriever_service import get_embeddings + +logging.basicConfig(level=logging.INFO) +logger = logging.getLogger(__name__) + + +def make_text_splitter() -> RecursiveCharacterTextSplitter: + """Chunking used for the program corpus (shared with scripts/seed_catalog.py).""" + return RecursiveCharacterTextSplitter( + chunk_size=1000, + chunk_overlap=200, + length_function=len, + ) + + +class DataIngestionService: + def __init__(self): + """Initialize the data ingestion service with ChromaDB and embeddings.""" + settings = get_settings() + self.data_dir = settings.catalog_data_dir + self.chroma_persist_dir = settings.chroma_persist_directory + self.collection_name = settings.chroma_collection_name + + # Initialize embeddings (same provider/model as the retriever) + self.embeddings = get_embeddings(settings) + + # Initialize ChromaDB client + self.client = chromadb.PersistentClient(path=self.chroma_persist_dir) + self.vector_store = Chroma( + client=self.client, + collection_name=self.collection_name, + embedding_function=self.embeddings, + ) + + # Text splitter for chunking documents + self.text_splitter = make_text_splitter() + + def get_program_files(self) -> List[str]: + """Get all program requirement file paths.""" + program_files = [] + for root, dirs, files in os.walk(self.data_dir): + for file in files: + if file == "requirements.txt": + program_files.append(os.path.join(root, file)) + return program_files + + def extract_program_metadata(self, file_path: str) -> Dict[str, Any]: + """Extract metadata from the file path and content.""" + # Get program name from directory structure + rel_path = os.path.relpath(file_path, self.data_dir) + dir_parts = rel_path.split(os.sep) + + # Program name is the directory containing requirements.txt + program_name = dir_parts[-2] if len(dir_parts) > 1 else "unknown" + + # Try to determine program type from name + program_type = "unknown" + if "bachelor_of_science" in program_name: + program_type = "bachelor_science" + elif "bachelor_of_arts" in program_name: + program_type = "bachelor_arts" + elif "certificate" in program_name: + program_type = "certificate" + elif "minor" in program_name: + program_type = "minor" + elif "double_major" in program_name: + program_type = "double_major" + + return { + "source": program_name, + "program_type": program_type, + "file_path": rel_path, + } + + def process_program_file(self, file_path: str) -> List[Document]: + """Process a single program file and convert it to documents.""" + try: + with open(file_path, 'r', encoding='utf-8') as f: + content = f.read() + + metadata = self.extract_program_metadata(file_path) + + # Create documents with metadata + document = Document( + page_content=content, + metadata=metadata + ) + + # Split into chunks + chunks = self.text_splitter.split_documents([document]) + return chunks + + except Exception as e: + logger.error(f"Error processing file {file_path}: {e}") + return [] + + def ingest_all_data(self) -> bool: + """Ingest all program data into ChromaDB.""" + try: + program_files = self.get_program_files() + logger.info(f"Found {len(program_files)} program files to process") + + all_chunks = [] + for file_path in program_files: + chunks = self.process_program_file(file_path) + all_chunks.extend(chunks) + logger.info(f"Processed {file_path}: {len(chunks)} chunks") + + if not all_chunks: + logger.error("No chunks were processed successfully") + return False + + # Add documents to ChromaDB + self.vector_store.add_documents(all_chunks) + logger.info(f"Successfully ingested {len(all_chunks)} document chunks into ChromaDB") + + # Test retrieval + test_results = self.vector_store.similarity_search("computer science requirements", k=2) + logger.info(f"Test retrieval successful: Found {len(test_results)} relevant documents") + + return True + + except Exception as e: + logger.error(f"Error during data ingestion: {e}") + return False + + def get_collection_info(self) -> Dict[str, Any]: + """Get information about the ChromaDB collection.""" + try: + collection = self.client.get_collection(self.collection_name) + return { + "collection_name": self.collection_name, + "document_count": collection.count(), + "metadata": collection.metadata or {} + } + except Exception as e: + logger.error(f"Error getting collection info: {e}") + return {"error": str(e)} + + +def main(): + """Main function to run data ingestion.""" + service = DataIngestionService() + + logger.info("Starting data ingestion...") + success = service.ingest_all_data() + + if success: + logger.info("Data ingestion completed successfully!") + info = service.get_collection_info() + logger.info(f"Collection info: {info}") + else: + logger.error("Data ingestion failed!") + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/backend/app/services/document_management.py b/backend/app/services/document_management.py new file mode 100644 index 0000000..3dfa489 --- /dev/null +++ b/backend/app/services/document_management.py @@ -0,0 +1,332 @@ +""" +Document management utilities for dashboard and batch operations. +Provides high-level helper functions for common document operations. +""" + +from typing import List, Dict, Any, Optional, Tuple +from datetime import datetime +from sqlalchemy.orm import Session +from sqlalchemy import func + +from app.db.models import Document, Chunk, Embedding +from app.db.services import DocumentService, ChunkService, EmbeddingService + + +class DocumentManagementUtils: + """Utility functions for document management and dashboard operations.""" + + @staticmethod + def get_document_statistics(db: Session, doc_id: str) -> Dict[str, Any]: + """Get comprehensive statistics for a document.""" + document = DocumentService.get_document(db=db, doc_id=doc_id) + if not document: + return {} + + chunks = ChunkService.list_chunks_by_document(db=db, document_id=doc_id, limit=10000) + embeddings = EmbeddingService.get_embeddings_by_document(db=db, document_id=doc_id) + + synced_embeddings = sum(1 for e in embeddings if e.is_synced) + unsynced_embeddings = sum(1 for e in embeddings if not e.is_synced) + + total_text_length = sum(len(chunk.text) for chunk in chunks) + avg_chunk_length = total_text_length / len(chunks) if chunks else 0 + + return { + "document_id": doc_id, + "title": document.title, + "source": document.source, + "created_at": document.created_at, + "updated_at": document.updated_at, + "chunk_count": len(chunks), + "embedding_count": len(embeddings), + "synced_embeddings": synced_embeddings, + "unsynced_embeddings": unsynced_embeddings, + "total_text_length": total_text_length, + "average_chunk_length": avg_chunk_length, + "sync_percentage": (synced_embeddings / len(embeddings) * 100) if embeddings else 0, + } + + @staticmethod + def get_all_documents_dashboard(db: Session) -> List[Dict[str, Any]]: + """Get dashboard view of all documents with key statistics.""" + documents = DocumentService.list_documents(db=db, limit=10000) + + dashboard_data = [] + for doc in documents: + chunks = ChunkService.list_chunks_by_document(db=db, document_id=doc.id, limit=10000) + embeddings = EmbeddingService.get_embeddings_by_document(db=db, document_id=doc.id) + + synced = sum(1 for e in embeddings if e.is_synced) + + dashboard_data.append({ + "id": doc.id, + "title": doc.title, + "source": doc.source, + "created_at": doc.created_at, + "updated_at": doc.updated_at, + "chunks": len(chunks), + "embeddings": len(embeddings), + "synced": synced, + "status": "synced" if len(embeddings) > 0 and synced == len(embeddings) else "partial" if synced > 0 else "unsynced", + }) + + return dashboard_data + + @staticmethod + def batch_delete_documents( + db: Session, doc_ids: List[str], hard_delete: bool = False + ) -> Dict[str, Any]: + """Batch delete multiple documents.""" + success_count = 0 + failed_ids = [] + + for doc_id in doc_ids: + try: + success = DocumentService.delete_document( + db=db, doc_id=doc_id, hard_delete=hard_delete + ) + if success: + success_count += 1 + else: + failed_ids.append(doc_id) + except Exception as e: + failed_ids.append(doc_id) + + return { + "deleted_count": success_count, + "failed_count": len(failed_ids), + "failed_ids": failed_ids, + "total": len(doc_ids), + } + + @staticmethod + def duplicate_document_with_chunks( + db: Session, source_doc_id: str, new_title: Optional[str] = None + ) -> Optional[Document]: + """Duplicate a document including all its chunks and embeddings.""" + source_doc = DocumentService.get_document(db=db, doc_id=source_doc_id) + if not source_doc: + return None + + # Create new document + new_doc = DocumentService.create_document( + db=db, + title=new_title or f"{source_doc.title} (Copy)", + source=source_doc.source, + metadata=source_doc.doc_metadata, + ) + + # Copy all chunks + source_chunks = ChunkService.list_chunks_by_document( + db=db, document_id=source_doc_id, limit=10000 + ) + + for chunk in source_chunks: + new_chunk = ChunkService.create_chunk( + db=db, + document_id=new_doc.id, + chunk_index=chunk.chunk_index, + text=chunk.text, + metadata=chunk.metadata, + ) + + # Copy embeddings + embeddings = EmbeddingService.list_embeddings_by_chunk( + db=db, chunk_id=chunk.id + ) + for embedding in embeddings: + EmbeddingService.create_embedding( + db=db, + chunk_id=new_chunk.id, + vector=embedding.vector, + model=embedding.model, + ) + + return new_doc + + @staticmethod + def export_document_to_json(db: Session, doc_id: str) -> Dict[str, Any]: + """Export document with chunks and embeddings to JSON format.""" + document = DocumentService.get_document(db=db, doc_id=doc_id) + if not document: + return {} + + chunks = ChunkService.list_chunks_by_document(db=db, document_id=doc_id, limit=10000) + + chunks_data = [] + for chunk in chunks: + embeddings = EmbeddingService.list_embeddings_by_chunk(db=db, chunk_id=chunk.id) + + chunk_data = { + "id": chunk.id, + "index": chunk.chunk_index, + "text": chunk.text, + "metadata": chunk.chunk_metadata, + "embeddings": [ + { + "id": emb.id, + "model": emb.model, + "chroma_id": emb.chroma_id, + "is_synced": emb.is_synced, + "vector_length": len(emb.vector) if emb.vector else 0, + } + for emb in embeddings + ], + } + chunks_data.append(chunk_data) + + return { + "document": { + "id": document.id, + "title": document.title, + "source": document.source, + "metadata": document.doc_metadata, + "created_at": document.created_at.isoformat(), + "updated_at": document.updated_at.isoformat(), + }, + "chunks": chunks_data, + "statistics": DocumentManagementUtils.get_document_statistics(db=db, doc_id=doc_id), + } + + @staticmethod + def search_content_across_documents( + db: Session, search_query: str, limit: int = 100 + ) -> List[Dict[str, Any]]: + """Search for content across all chunks in all documents.""" + results = [] + + # Search in chunk text + chunks = ( + db.query(Chunk) + .filter( + Chunk.text.ilike(f"%{search_query}%"), + Chunk.is_deleted == False, + ) + .limit(limit) + .all() + ) + + for chunk in chunks: + document = DocumentService.get_document(db=db, doc_id=chunk.document_id) + if document: + results.append({ + "type": "chunk", + "document_id": document.id, + "document_title": document.title, + "chunk_id": chunk.id, + "chunk_index": chunk.chunk_index, + "text_preview": chunk.text[:200], # First 200 chars + "full_text": chunk.text, + }) + + # Search in document titles + documents = DocumentService.search_documents(db=db, query_str=search_query) + for doc in documents: + results.append({ + "type": "document", + "document_id": doc.id, + "document_title": doc.title, + "source": doc.source, + }) + + return results + + @staticmethod + def get_sync_status_summary(db: Session) -> Dict[str, Any]: + """Get overall sync status of all embeddings to ChromaDB.""" + total_embeddings = db.query(func.count(Embedding.id)).scalar() or 0 + synced_embeddings = ( + db.query(func.count(Embedding.id)) + .filter(Embedding.is_synced == True) + .scalar() + or 0 + ) + unsynced_embeddings = total_embeddings - synced_embeddings + + total_documents = ( + db.query(func.count(Document.id)) + .filter(Document.is_deleted == False) + .scalar() + or 0 + ) + total_chunks = ( + db.query(func.count(Chunk.id)) + .filter(Chunk.is_deleted == False) + .scalar() + or 0 + ) + + return { + "total_documents": total_documents, + "total_chunks": total_chunks, + "total_embeddings": total_embeddings, + "synced_embeddings": synced_embeddings, + "unsynced_embeddings": unsynced_embeddings, + "sync_percentage": (synced_embeddings / total_embeddings * 100) if total_embeddings > 0 else 0, + "last_updated": datetime.utcnow(), + } + + @staticmethod + def get_recent_activity(db: Session, days: int = 7, limit: int = 100) -> Dict[str, List]: + """Get recent document and chunk activity.""" + from datetime import timedelta + + cutoff_date = datetime.utcnow() - timedelta(days=days) + + recent_documents = ( + db.query(Document) + .filter( + Document.updated_at >= cutoff_date, + Document.is_deleted == False, + ) + .order_by(Document.updated_at.desc()) + .limit(limit) + .all() + ) + + recent_chunks = ( + db.query(Chunk) + .filter( + Chunk.updated_at >= cutoff_date, + Chunk.is_deleted == False, + ) + .order_by(Chunk.updated_at.desc()) + .limit(limit) + .all() + ) + + recent_embeddings = ( + db.query(Embedding) + .filter(Embedding.updated_at >= cutoff_date) + .order_by(Embedding.updated_at.desc()) + .limit(limit) + .all() + ) + + return { + "documents": [ + { + "id": doc.id, + "title": doc.title, + "updated_at": doc.updated_at, + } + for doc in recent_documents + ], + "chunks": [ + { + "id": chunk.id, + "document_id": chunk.document_id, + "updated_at": chunk.updated_at, + } + for chunk in recent_chunks + ], + "embeddings": [ + { + "id": emb.id, + "chunk_id": emb.chunk_id, + "updated_at": emb.updated_at, + "is_synced": emb.is_synced, + } + for emb in recent_embeddings + ], + } diff --git a/backend/app/services/embedding_service.py b/backend/app/services/embedding_service.py new file mode 100644 index 0000000..26f9e0d --- /dev/null +++ b/backend/app/services/embedding_service.py @@ -0,0 +1,55 @@ +from fastapi import FastAPI +from pydantic import BaseModel +from langchain_community.embeddings import HuggingFaceEmbeddings +import chromadb + +# --- Initialize app --- +app = FastAPI(title="Embeddings Service") + +# --- Initialize embedding model and Chroma vector store --- +embedding_model = HuggingFaceEmbeddings(model_name="sentence-transformers/all-MiniLM-L6-v2") + +# persist_directory means Chroma will save data in /db/chroma/ +client = chromadb.Client() +collection = client.get_or_create_collection(name="chunks") + +# --- Define request models --- +class EmbedItem(BaseModel): + chunk_id: str + text: str + metadata: dict = {} + +class EmbedBatch(BaseModel): + items: list[EmbedItem] + +class QueryRequest(BaseModel): + query: str + top_k: int = 5 + + +# --- Endpoints --- + +@app.get("/") +def root(): + """Health check endpoint""" + return {"message": "Embeddings service is running 🚀"} + +@app.post("/embed_batch") +def embed_batch(req: EmbedBatch): + """Store text chunks as vectors in Chroma""" + texts = [i.text for i in req.items] + ids = [i.chunk_id for i in req.items] + metadatas = [i.metadata for i in req.items] + + vectors = embedding_model.embed_documents(texts) + collection.add(ids=ids, embeddings=vectors, metadatas=metadatas, documents=texts) + + return {"status": "ok", "count": len(vectors)} + +@app.post("/search") +def search(req: QueryRequest): + """Search for the most similar chunks to a given query""" + query_vec = embedding_model.embed_query(req.query) + results = collection.query(query_embeddings=[query_vec], n_results=req.top_k) + return {"results": results} + diff --git a/backend/app/services/general_webscraper.py b/backend/app/services/general_webscraper.py new file mode 100644 index 0000000..f2e1df8 --- /dev/null +++ b/backend/app/services/general_webscraper.py @@ -0,0 +1,250 @@ +import asyncio +import json +import os +from pathlib import Path +from urllib.parse import urlparse +from datetime import datetime +from crawl4ai import AsyncWebCrawler, CrawlerRunConfig, BrowserConfig +from crawl4ai.deep_crawling import BFSDeepCrawlStrategy, DomainFilter, FilterChain, ContentTypeFilter, URLPatternFilter +from crawl4ai.content_scraping_strategy import LXMLWebScrapingStrategy +from collections import defaultdict + +import logging + +logging.basicConfig( + level=logging.INFO, + format='%(asctime)s - %(levelname)s - %(message)s' +) +logger = logging.getLogger(__name__) + +class WebCrawlingPipeline: + def __init__(self, output_folder: str = "output"): + self.output_folder = Path(output_folder) + self.output_folder.mkdir(exist_ok=True) + self.index_file = self.output_folder / "index.json" + self.visited_urls = self._load_visited_urls() + self.failed_domains = defaultdict(int) + + def _load_visited_urls(self) -> set: + """Load previously visited URLs from index file""" + if self.index_file.exists(): + with open(self.index_file, 'r') as f: + data = json.load(f) + return set(data.get('visited_urls', [])) + return set() + + def _save_index(self, results: list): + """Save index file with all crawled URLs""" + index_data = { + 'last_updated': datetime.now().isoformat(), + 'total_pages': len(results), + 'visited_urls': list(self.visited_urls), + 'pages': [] + } + + for result in results: + if result.success: + filename = self._get_filename(result.url) + index_data['pages'].append({ + 'url': result.url, + 'filename': filename, + 'status_code': result.status_code, + 'depth': result.metadata.get('depth', 0) if result.metadata else 0 + }) + + with open(self.index_file, 'w') as f: + json.dump(index_data, f, indent=2) + logger.info(f"Index file saved: {self.index_file}") + + def _get_filename(self, url: str) -> str: + """Generate safe filename from URL""" + parsed = urlparse(url) + path = parsed.path.strip('/').replace('/', '_') or 'index' + return f"{parsed.netloc}_{path}" + + def _save_page_data(self, result): + """Save all page data (HTML, markdown, and metadata) in a single JSON file""" + if not result.success: + error_msg = result.error_message or "" + if "ERR_ABORTED" in error_msg or "Target page, context or browser has been closed" in error_msg: + logger.debug(f"Skipping {result.url} - limit reached during processing") + return + + domain = urlparse(result.url).netloc + self.failed_domains[domain] += 1 + logger.error(f"Failed to crawl {result.url}: {error_msg}") + + + filename = self._get_filename(result.url) + + # Extract markdown content + markdown_content = result.markdown.raw_markdown if hasattr(result.markdown, 'raw_markdown') else str(result.markdown) + + # Consolidate all data into a single JSON structure + page_data = { + 'url': result.url, + 'status_code': result.status_code, + 'crawled_at': datetime.now().isoformat(), + 'depth': result.metadata.get('depth', 0) if result.metadata else 0, + 'title': result.metadata.get('title') if result.metadata else None, + 'html': result.html, + 'markdown': markdown_content, + 'cleaned_html': result.cleaned_html, + 'links': { + 'internal': result.links.get('internal', []), + 'external': result.links.get('external', []) + }, + 'media': result.media, + 'metadata': result.metadata + } + + # Save single JSON file + json_file = self.output_folder / f"{filename}.json" + with open(json_file, 'w', encoding='utf-8') as f: + json.dump(page_data, f, indent=2, ensure_ascii=False) + logger.info(f"Saved all data to: {json_file}") + + self.visited_urls.add(result.url) + + def create_generic_filter_chain(self, base_url: str, allow_subdomains: bool = True, blocked_patterns: list[str] = None, blocked_domains: list[str] = None, allowed_content_types: list[str] = None): + """ + Create a generic filter chain that works for any website. + + Args: + base_url: Starting URL to extract domain from + allow_subdomains: Whether to allow subdomains of the base domain + blocked_patterns: URL patterns to exclude (e.g., ["*.zip", "*.pdf"]) + allowed_content_types: Content types to allow (default: ["text/html"]) + """ + # Extract domain from base URL + parsed = urlparse(base_url) + base_domain = parsed.netloc + + # Default blocked patterns for common non-HTML resources + if blocked_patterns is None: + blocked_patterns = [ + "*.zip", "*.pdf", "*.exe", "*.tar.gz", "*.dmg", + "*.jpg", "*.jpeg", "*.png", "*.gif", "*.svg", + "*.mp4", "*.mp3", "*.avi", "*.mov" + ] + + # Default to HTML content only + if allowed_content_types is None: + allowed_content_types = ["text/html", "application/xhtml+xml"] + + filters = [] + + # Domain filter - allow base domain and optionally subdomains + if allow_subdomains: + # Use wildcard pattern to match subdomains + allowed_domains = [f"*.{base_domain}", base_domain] + else: + allowed_domains = [base_domain] + + filters.append(DomainFilter(allowed_domains=allowed_domains, blocked_domains=blocked_domains or [])) + + # Content type filter + filters.append(ContentTypeFilter(allowed_types=allowed_content_types)) + + # URL pattern filter to exclude file downloads + if blocked_patterns: + filters.append(URLPatternFilter( + patterns=blocked_patterns, + reverse=True # Exclude matches + )) + + return FilterChain(filters) + + async def crawl(self, urls: list[str] | str, max_depth: int = 2, max_pages: int = 50, timeout: int = 120): + """Main crawling function""" + if isinstance(urls, str): + urls = [urls] + + # Filter out already visited URLs + urls_to_crawl = [url for url in urls if url not in self.visited_urls] + if not urls_to_crawl: + logger.info("All URLs already visited") + return + + # Extract domain for filtering + base_domain = urlparse(urls_to_crawl[0]).netloc + + # Create generic filter chain automatically + filter_chain = self.create_generic_filter_chain( + base_url=urls_to_crawl[0], + allow_subdomains=True, # Allow docs.example.com, blog.example.com, etc. + blocked_patterns=None, # Use defaults + blocked_domains=[], + allowed_content_types=None # Use defaults + ) + + deep_crawl_strategy = BFSDeepCrawlStrategy( + max_depth=max_depth, + include_external=False, + max_pages=max_pages, + filter_chain=filter_chain + ) + + # Configure crawler + browser_config = BrowserConfig( + headless=True, + verbose=True, + # text_mode=True # Skips image dimension updates + ) + + run_config = CrawlerRunConfig( + deep_crawl_strategy=deep_crawl_strategy, + scraping_strategy=LXMLWebScrapingStrategy(), + stream=True, + verbose=True, + page_timeout=90000, + word_count_threshold=10, + wait_until="load", + simulate_user=True, + delay_before_return_html=3.0 + ) + + results = [] + start_time = datetime.now() + last_result_time = start_time + async with AsyncWebCrawler(config=browser_config) as crawler: + logger.info(f"Starting crawl of {len(urls_to_crawl)} URLs with max_depth={max_depth}") + + try: + # Stream results with explicit completion handling + async def crawl_task(): + async for result in await crawler.arun(urls_to_crawl[0], config=run_config): + logger.info(f"Processing: {result.url} (Depth: {result.metadata.get('depth', 0) if result.metadata else 0})") + + if not result.success: + error_msg = result.error_message or "" + if any(err in error_msg for err in [ + "ERR_ABORTED", + "Target page, context or browser has been closed", + "ERR_HTTP_RESPONSE_CODE_FAILURE" # Server-side errors + ]): + logger.debug(f"Skipping {result.url} - expected error during cleanup or server issue") + continue + + self._save_page_data(result) + results.append(result) + + # Apply timeout to the task, not the context manager + await asyncio.wait_for(crawl_task(), timeout=timeout) + + except asyncio.TimeoutError: + logger.warning(f"Crawl timed out after {timeout}s. Processed {len(results)} pages") + except StopAsyncIteration: + logger.info("Stream completed normally") + except Exception as e: + logger.error(f"Error during crawl: {e}") + finally: + elapsed = (datetime.now() - start_time).total_seconds() + logger.info(f"Crawl finished. Total results: {len(results)}, elapsed: {elapsed:.1f}s") + + # Save index file + self._save_index(results) + logger.info(f"Crawl complete. Processed {len(results)} pages") + + return results + \ No newline at end of file diff --git a/backend/app/services/housing_scraper.py b/backend/app/services/housing_scraper.py new file mode 100644 index 0000000..6eb472b --- /dev/null +++ b/backend/app/services/housing_scraper.py @@ -0,0 +1,288 @@ +""" +UTD Housing Scraper +Scrapes housing information from housing.utdallas.edu including all internal subpages. +""" + +import asyncio +import json +import logging +import re +from pathlib import Path +from typing import Optional +from urllib.parse import urljoin, urlparse + +from playwright.async_api import ( + async_playwright, + Browser, + Page, + TimeoutError as PlaywrightTimeoutError, +) + +logging.basicConfig( + level=logging.INFO, + format="%(asctime)s - %(levelname)s - %(message)s", +) +logger = logging.getLogger(__name__) + + +class UTDHousingScraper: + """Scrapes housing.utdallas.edu and all its internal subpages.""" + + BASE_URL = "https://housing.utdallas.edu/" + DOMAIN = "housing.utdallas.edu" + + def __init__( + self, + max_depth: int = 2, + max_pages: Optional[int] = None, + rate_limit: float = 1.0, + max_parallel: int = 3, + output_dir: str = "./output", + ): + self.max_depth = max_depth + self.max_pages = max_pages + self.rate_limit = rate_limit + self.max_parallel = max_parallel + self.output_dir = Path(output_dir) + self.output_dir.mkdir(parents=True, exist_ok=True) + self._last_request_time = 0 + self.visited_urls: set[str] = set() + self.results: list[dict] = [] + + async def _rate_limit_wait(self): + """Apply rate limiting between requests.""" + current_time = asyncio.get_event_loop().time() + time_since_last = current_time - self._last_request_time + if time_since_last < self.rate_limit: + await asyncio.sleep(self.rate_limit - time_since_last) + self._last_request_time = asyncio.get_event_loop().time() + + @staticmethod + def _sanitize_filename(name: str) -> str: + """Convert a name to a safe filename.""" + name = re.sub(r"[^\w\s-]", "", name) + name = re.sub(r"[-\s]+", "_", name) + return name.strip("_").lower() or "index" + + def _normalize_url(self, url: str) -> str: + """Normalize a URL by removing fragments and trailing slashes.""" + parsed = urlparse(url) + # Rebuild without fragment + normalized = f"{parsed.scheme}://{parsed.netloc}{parsed.path}" + if parsed.query: + normalized += f"?{parsed.query}" + return normalized.rstrip("/") + + def _is_valid_link(self, url: str) -> bool: + """Check if a URL is a valid internal housing page link.""" + parsed = urlparse(url) + + # Must be on the housing domain + if parsed.netloc and parsed.netloc != self.DOMAIN: + return False + + # Skip non-HTML resources + skip_extensions = { + ".pdf", ".jpg", ".jpeg", ".png", ".gif", ".svg", + ".mp4", ".mp3", ".zip", ".exe", ".doc", ".docx", + ".xls", ".xlsx", ".ppt", ".pptx", + } + path_lower = parsed.path.lower() + if any(path_lower.endswith(ext) for ext in skip_extensions): + return False + + # Skip mailto and tel links + if url.startswith(("mailto:", "tel:", "javascript:")): + return False + + return True + + async def _extract_links(self, page: Page, current_url: str) -> list[str]: + """Extract all valid internal links from a page.""" + links = [] + try: + anchors = await page.query_selector_all("a[href]") + for anchor in anchors: + href = await anchor.get_attribute("href") + if not href: + continue + full_url = urljoin(current_url, href) + normalized = self._normalize_url(full_url) + if self._is_valid_link(normalized) and normalized not in self.visited_urls: + links.append(normalized) + except Exception as e: + logger.warning(f"Error extracting links from {current_url}: {e}") + return list(set(links)) + + async def _scrape_page(self, page: Page, url: str) -> Optional[dict]: + """Scrape content from a single page.""" + try: + await page.goto(url, wait_until="networkidle", timeout=30000) + await self._rate_limit_wait() + + # Get the page title + title = await page.title() + + # Extract main content, excluding nav/header/footer + content = await page.evaluate(""" + () => { + // Remove noisy elements + const remove = document.querySelectorAll( + 'script, style, nav, header, footer, .menu, #menu, .nav, .navbar' + ); + remove.forEach(el => el.remove()); + + const main = document.querySelector( + 'main, .main-content, #content, .content, .entry-content, article, .page-content' + ); + return main ? main.innerText : document.body.innerText; + } + """) + + return { + "url": url, + "title": title.strip() if title else "", + "content": content.strip() if content else "", + } + + except PlaywrightTimeoutError: + logger.error(f"Timeout loading: {url}") + return None + except Exception as e: + logger.error(f"Error scraping {url}: {e}") + return None + + async def _scrape_and_extract(self, browser: Browser, url: str) -> tuple[Optional[dict], list[str]]: + """Scrape a single page and extract its links. Returns (page_data, child_links).""" + context = await browser.new_context() + page = await context.new_page() + child_links = [] + data = None + try: + data = await self._scrape_page(page, url) + # Re-navigate to get links (scrape_page modifies the DOM) + await page.goto(url, wait_until="networkidle", timeout=30000) + child_links = await self._extract_links(page, url) + except Exception as e: + logger.error(f"Error processing {url}: {e}") + finally: + await context.close() + return data, child_links + + async def _bfs_crawl(self, browser: Browser): + """BFS crawl that processes pages level-by-level to avoid deadlocks.""" + # Queue holds (url, depth) tuples + queue: list[tuple[str, int]] = [(self.BASE_URL, 0)] + self.visited_urls.add(self._normalize_url(self.BASE_URL)) + + while queue: + if self.max_pages and len(self.results) >= self.max_pages: + break + + # Process current batch concurrently (up to max_parallel at a time) + batch = queue[:] + queue.clear() + + # Process batch in chunks of max_parallel + for i in range(0, len(batch), self.max_parallel): + if self.max_pages and len(self.results) >= self.max_pages: + break + + chunk = batch[i : i + self.max_parallel] + tasks = [self._scrape_and_extract(browser, url) for url, _ in chunk] + results = await asyncio.gather(*tasks, return_exceptions=True) + + for (url, depth), result in zip(chunk, results): + if isinstance(result, Exception): + logger.error(f"Error crawling {url}: {result}") + continue + + data, child_links = result + if data: + data["depth"] = depth + self.results.append(data) + logger.info( + f"[depth={depth}] Scraped: {data['title'] or url} " + f"({len(data['content'])} chars)" + ) + + # Queue child links if within depth limit + if depth < self.max_depth: + for link in child_links: + normalized = self._normalize_url(link) + if normalized not in self.visited_urls: + if self.max_pages and len(self.visited_urls) >= self.max_pages: + break + self.visited_urls.add(normalized) + queue.append((link, depth + 1)) + logger.info(f" Queued {len(child_links)} links from depth {depth}") + + # Rate limit between chunks + await self._rate_limit_wait() + + def _save_results(self): + """Save all scraped results to disk.""" + housing_dir = self.output_dir / "housing" + housing_dir.mkdir(parents=True, exist_ok=True) + + for data in self.results: + # Build a filename from the URL path + parsed = urlparse(data["url"]) + path_part = parsed.path.strip("/") + if not path_part: + filename = "index" + else: + filename = self._sanitize_filename(path_part.replace("/", "_")) + + # Save content as text file + txt_file = housing_dir / f"{filename}.txt" + header = f"URL: {data['url']}\nTitle: {data['title']}\nDepth: {data['depth']}\n" + txt_file.write_text( + f"{header}\n{'-' * 60}\n\n{data['content']}", + encoding="utf-8", + ) + + # Save an index JSON with metadata + index = { + "source": self.BASE_URL, + "total_pages": len(self.results), + "pages": [ + {"url": r["url"], "title": r["title"], "depth": r["depth"]} + for r in self.results + ], + } + index_file = housing_dir / "index.json" + with open(index_file, "w") as f: + json.dump(index, f, indent=2) + + logger.info(f"Saved {len(self.results)} pages to {housing_dir}") + + async def scrape(self): + """Main entry point — launches browser and starts crawling.""" + async with async_playwright() as p: + browser = await p.chromium.launch(headless=True) + try: + await self._bfs_crawl(browser) + finally: + await browser.close() + + self._save_results() + return self.results + + +async def main(): + scraper = UTDHousingScraper( + max_depth=2, + max_pages=50, + rate_limit=1.5, + max_parallel=3, + output_dir=Path(__file__).parent / "data", + ) + results = await scraper.scrape() + print(f"\nDone! Scraped {len(results)} pages.") + for r in results: + print(f" [{r['depth']}] {r['title'] or r['url']}") + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/backend/app/services/html_processing_pipeline.py b/backend/app/services/html_processing_pipeline.py new file mode 100644 index 0000000..ae95a0f --- /dev/null +++ b/backend/app/services/html_processing_pipeline.py @@ -0,0 +1,244 @@ +import json +import logging +from io import BytesIO +from pathlib import Path +from typing import Any, Dict, List, Optional, Union +from datetime import datetime +from pydantic import BaseModel, Field, field_validator, ConfigDict +from docling.backend.html_backend import HTMLDocumentBackend +from docling.datamodel.base_models import InputFormat +from docling.datamodel.document import InputDocument +from docling.chunking import HybridChunker +from docling_core.types.doc import DoclingDocument + +from transformers import AutoTokenizer + +logging.basicConfig(level=logging.INFO) +logger = logging.getLogger(__name__) + +class ChunkMetadata(BaseModel): + """Metadata for a chunk of processed content.""" + document_name: str = Field(..., description="Name of the source document") + headings: List[str] = Field(default_factory=list, description="Hierarchical headings context") + doc_items: List[str] = Field(default_factory=list, description="Document item references") + origin: Optional[Dict[str, str]] = Field(None, description="Origin information (filename, mimetype)") + + model_config = ConfigDict(extra="allow") + + +class ChunkResult(BaseModel): + """Represents a single chunk of processed content.""" + content: str = Field(..., min_length=1, description="The chunked text content") + source: str = Field(..., description="Origin of the content (file path, raw HTML, or JSON)") + metadata: Optional[ChunkMetadata] = Field(None, description="Additional metadata about the chunk") + + @field_validator('content') + @classmethod + def validate_content_not_empty(cls, v: str) -> str: + """Ensure content is not just whitespace.""" + if not v.strip(): + raise ValueError("Content cannot be empty or whitespace-only") + return v + + +class PipelineConfig(BaseModel): + """Configuration for the HTML processing pipeline.""" + chunker: Optional[HybridChunker] = Field(None, description="Custom chunker instance") + validate_html: bool = Field(True, description="Whether to validate HTML structure") + min_chunk_length: int = Field(1, ge=1, description="Minimum chunk length in characters") + + model_config = ConfigDict(arbitrary_types_allowed=True) + + +class HTMLInput(BaseModel): + """Validated input for HTML processing.""" + + html_content: str = Field(..., min_length=1, description="HTML content to process") + source: str = Field(..., description="Source identifier") + + @field_validator('html_content') + @classmethod + def validate_html_tags(cls, v: str) -> str: + """Basic validation that content looks like HTML.""" + if '<' not in v or '>' not in v: + raise ValueError("Content does not appear to be valid HTML") + return v + + +class HTMLProcessingPipeline: + """ + Flexible pipeline for processing HTML from multiple input formats. + Handles HTML files, raw HTML text, and JSON objects with HTML content. + """ + + def __init__(self, config: Optional[PipelineConfig] = None): + """ + Initialize the pipeline. + + Args: + config: Optional PipelineConfig instance. If None, uses defaults. + """ + self.config = config or PipelineConfig() + self.chunker = self.config.chunker or HybridChunker() + logger.info("HTMLProcessingPipeline initialized with Pydantic models") + + def process( + self, + input_data: Union[str, Path, Dict[str, Any]], + source_name: Optional[str] = None + ) -> List[ChunkResult]: + """ + Process HTML content from various input formats. + + Args: + input_data: Can be: + - Path or str (file path) + - str (raw HTML text) + - Dict with 'html' field (JSON object) + source_name: Optional name to identify the source + + Returns: + List of validated ChunkResult objects containing chunked content + + Raises: + ValueError: If input format is invalid or HTML content is missing + ValidationError: If Pydantic validation fails + """ + try: + html_input = self._extract_and_validate_html(input_data, source_name) + + doc = self._html_to_document(html_input.html_content, html_input.source) + + chunks = self._chunk_document(doc, html_input.source) + + logger.info(f"Successfully processed {len(chunks)} chunks from {html_input.source}") + return chunks + + except Exception as e: + logger.error(f"Pipeline failed for source '{source_name}': {str(e)}") + raise + + def _extract_and_validate_html( + self, + input_data: Union[str, Path, Dict[str, Any]], + source_name: Optional[str] + ) -> HTMLInput: + """ + Extract HTML content and validate using Pydantic. + + Returns: + Validated HTMLInput object + """ + # Case 1: Dictionary/JSON input + if isinstance(input_data, dict): + html_content = input_data.get('cleaned_html', '') + if not html_content: + html_content = input_data.get('html', '') + if not html_content: + html_content = input_data.get('content', '') or input_data.get('html_content', '') + if not html_content: + raise ValueError("JSON object missing 'cleaned_html', 'html' field or valid fallback") + + datetime_value = input_data.get('crawled_at') + + source = source_name or "JSON object" + logger.info(f"Extracted HTML from JSON object (length: {len(html_content)})") + + return html_content, source, datetime_value + + # Case 2: File path + if isinstance(input_data, (Path, str)): + path = Path(input_data) + + if path.exists() and path.is_file(): + html_content = path.read_text(encoding='utf-8') + source = str(path) + logger.info(f"Read HTML from file: {source}") + + return html_content, source, datetime.now().isoformat() + + # Case 3: Raw HTML text (not a valid file path) + html_content = str(input_data) + source = source_name or "raw HTML text" + logger.info(f"Processing raw HTML text (length: {len(html_content)})") + + return html_content, source, datetime.now().isoformat() + + raise ValueError(f"Unsupported input type: {type(input_data)}") + + def _html_to_document(self, html_content: str, source: str) -> DoclingDocument: + """ + Convert HTML content to DoclingDocument using HTMLDocumentBackend. + """ + try: + html_bytes = html_content.encode('utf-8') + stream = BytesIO(html_bytes) + + in_doc = InputDocument( + path_or_stream=stream, + format=InputFormat.HTML, + backend=HTMLDocumentBackend, + filename=source + ) + + backend = HTMLDocumentBackend( + in_doc=in_doc, + path_or_stream=stream + ) + + if self.config.validate_html and not backend.is_valid(): + raise ValueError(f"Invalid HTML document from source: {source}") + + doc = backend.convert() + logger.info(f"Converted HTML to DoclingDocument: {doc.name}") + return doc + + except Exception as e: + logger.error(f"Failed to convert HTML to document: {str(e)}") + raise + + def _chunk_document( + self, + doc: DoclingDocument, + source: str, + datetime_value: Optional[str] = None + ) -> List[ChunkResult]: + """ + Chunk the DoclingDocument into validated ChunkResult objects. + """ + try: + chunks = [] + for chunk in self.chunker.chunk(doc): + metadata = { + 'document_name': doc.name, + 'headings': chunk.meta.headings if hasattr(chunk.meta, 'headings') else [], + 'doc_items': [str(item) for item in (chunk.meta.doc_items if hasattr(chunk.meta, 'doc_items') else [])] + } + + if doc.origin: + metadata['origin'] = { + 'filename': doc.origin.filename, + 'mimetype': doc.origin.mimetype + } + + if datetime_value: + metadata['datetime'] = datetime_value + + + chunk_result = ChunkResult( + content=chunk.text, + source=source, + metadata=metadata + ) + + if len(chunk_result.content) >= self.config.min_chunk_length: + chunks.append(chunk_result) + else: + logger.debug(f"Skipping chunk below minimum length: {len(chunk_result.content)} chars") + + return chunks + + except Exception as e: + logger.error(f"Failed to chunk document: {str(e)}") + raise + \ No newline at end of file diff --git a/backend/app/services/ingest_service.py b/backend/app/services/ingest_service.py new file mode 100644 index 0000000..1def681 --- /dev/null +++ b/backend/app/services/ingest_service.py @@ -0,0 +1,216 @@ +import hashlib +from html.parser import HTMLParser +from typing import Optional, List, Dict, Any + +class _HTMLTextExtractor(HTMLParser): + """Simple HTML to text extractor using the standard library.""" + + def __init__(self) -> None: + super().__init__() + self._chunks: list[str] = [] + + def handle_data(self, data: str) -> None: + if data: + self._chunks.append(data) + + def get_text(self) -> str: + return " ".join(part.strip() for part in self._chunks if part.strip()) + + +class IngestService(): + def __init__(self, chunk_size: int = 1000, chunk_overlap: int = 200): + """ + Initialize the IngestService. + + Args: + chunk_size: Maximum size of text chunks in characters (default: 1000). + chunk_overlap: Number of characters to overlap between chunks (default: 200). + """ + self.chunk_size = chunk_size + self.chunk_overlap = chunk_overlap + + def _split_text(self, text: str) -> List[str]: + """ + Split text into chunks with overlap. + + Args: + text: Text to split. + + Returns: + List of text chunks. + """ + if len(text) <= self.chunk_size: + return [text] + + chunks: List[str] = [] + start = 0 + + while start < len(text): + end = start + self.chunk_size + chunk = text[start:end] + + # Try to break at sentence boundary if not at the end + if end < len(text): + # Look for sentence endings near the end + for break_char in ['. ', '.\n', '! ', '!\n', '? ', '?\n']: + last_break = chunk.rfind(break_char) + if last_break > self.chunk_size // 2: # Only break if reasonable + chunk = chunk[:last_break + 1] + end = start + len(chunk) + break + + chunks.append(chunk.strip()) + start = end - self.chunk_overlap + + return chunks + + def _create_chunks(self, text: str, source_url: str, base_chunk_id: str = "1") -> List[Dict[str, Any]]: + """ + Create chunk dictionaries from text. + + Args: + text: Text to chunk. + source_url: Source URL for the chunks. + base_chunk_id: Base ID for chunk numbering. + + Returns: + List of chunk dictionaries with chunk_id, text, and source_url. + """ + text_chunks = self._split_text(text) + chunks: List[Dict[str, Any]] = [] + + for i, chunk_text in enumerate(text_chunks): + if not chunk_text.strip(): + continue + + chunk_id = f"{base_chunk_id}_{i+1}" if len(text_chunks) > 1 else base_chunk_id + chunks.append({ + "chunk_id": chunk_id, + "text": chunk_text, + "source_url": source_url + }) + + return chunks + + def process_HTML(self, html_content: str, source_url: Optional[str] = None) -> List[Dict[str, Any]]: + """ + Convert basic HTML to plain text and return as JSON chunks. + + Args: + html_content: The HTML string to process. + source_url: Optional source URL. If not provided, generates a hash-based URL. + + Returns: + List of dictionaries with chunk_id, text, and source_url fields. + """ + parser = _HTMLTextExtractor() + parser.feed(html_content) + text = parser.get_text() + + if not source_url: + # Generate a hash-based identifier + content_hash = hashlib.md5(html_content.encode()).hexdigest()[:8] + source_url = f"html://content/{content_hash}" + + return self._create_chunks(text, source_url, base_chunk_id="html") + + def process_html_from_url(self, url: str, timeout: int = 30) -> List[Dict[str, Any]]: + """ + Fetch HTML content from a URL and return as JSON chunks. + + Args: + url: The URL to fetch HTML content from. + timeout: Request timeout in seconds (default: 30). + + Returns: + List of dictionaries with chunk_id, text, and source_url fields. + + Raises: + requests.RequestException: If the request fails (connection error, timeout, etc.). + RuntimeError: If requests library is not installed. + """ + try: + import requests # type: ignore + except ImportError: + raise RuntimeError( + "requests is required for URL HTML ingestion. Add 'requests' to requirements and install." + ) + + try: + response = requests.get(url, timeout=timeout) + response.raise_for_status() # Raises an HTTPError for bad responses + html_content = response.text + except requests.RequestException as e: + raise requests.RequestException( + f"Failed to fetch HTML from URL '{url}': {str(e)}" + ) from e + + # Use the URL as the source_url + return self.process_HTML(html_content, source_url=url) + + def process_pdf(self, file_path: str, source_url: Optional[str] = None) -> List[Dict[str, Any]]: + """ + Extract text from a PDF file and return as JSON chunks. + + Args: + file_path: Path to the PDF file on disk. + source_url: Optional source URL. If not provided, uses file:// path. + + Returns: + List of dictionaries with chunk_id, text, and source_url fields. + + Raises: + RuntimeError: If PyPDF2 is not installed. + """ + try: + import PyPDF2 # type: ignore + except Exception as exc: + raise RuntimeError( + "PyPDF2 is required for PDF ingestion. Add 'PyPDF2' to requirements and install." + ) + + if not source_url: + source_url = f"file://{file_path}" + + text_chunks: list[str] = [] + with open(file_path, "rb") as f: + reader = PyPDF2.PdfReader(f) + for page in reader.pages: + page_text: Optional[str] = page.extract_text() + if page_text: + text_chunks.append(page_text) + + full_text = "\n".join(text_chunks) + return self._create_chunks(full_text, source_url, base_chunk_id="pdf") + + def process_word(self, file_path: str, source_url: Optional[str] = None) -> List[Dict[str, Any]]: + """ + Extract text from a DOCX file and return as JSON chunks. + + Args: + file_path: Path to the .docx file on disk. + source_url: Optional source URL. If not provided, uses file:// path. + + Returns: + List of dictionaries with chunk_id, text, and source_url fields. + + Raises: + RuntimeError: If python-docx is not installed. + """ + try: + import docx # type: ignore + except Exception as exc: + raise RuntimeError( + "python-docx is required for DOCX ingestion. Add 'python-docx' to requirements and install." + ) from exc + + if not source_url: + source_url = f"file://{file_path}" + + document = docx.Document(file_path) + paragraphs = [p.text.strip() for p in document.paragraphs if p.text and p.text.strip()] + full_text = "\n".join(paragraphs) + + return self._create_chunks(full_text, source_url, base_chunk_id="docx") + + \ No newline at end of file diff --git a/backend/app/services/llm_service.py b/backend/app/services/llm_service.py new file mode 100644 index 0000000..5e4408b --- /dev/null +++ b/backend/app/services/llm_service.py @@ -0,0 +1,19 @@ +from langchain_ollama import ChatOllama +import random + + +class LLMService: + def __init__(self): + self.llm = ChatOllama(model="llama3.1:8b", temperature=0.4) + + def call(self): + # note that this is just a place holder, this class is only for calling the LLM + countries = ["Indonesia", "Germany", "China", "France", "Japan", "Brazil"] + content = self.llm.invoke(f"What is the capital of {random.choice(countries)}?").content + return content + + async def a_call(self): + # Just know that this is an async call to the LLM hence the 'a_' prefix + countries = ["Indonesia", "Germany", "China", "France", "Japan", "Brazil"] + content = await self.llm.ainvoke(f"What is the capital of {random.choice(countries)}?") + return content.content diff --git a/backend/app/services/prompt_service.py b/backend/app/services/prompt_service.py new file mode 100644 index 0000000..5a3799d --- /dev/null +++ b/backend/app/services/prompt_service.py @@ -0,0 +1,21 @@ +from langchain_core.prompts import PromptTemplate + +template = """ +You are an expert at answering questions about The University of Texas at Dallas (UTD). +Your goal is to provide accurate and helpful information to students, faculty, and staff. + +Use the following pieces of retrieved context to answer the question. +If you don't know the answer, just say that you don't know. +Do not make up an answer. +Cite the source of your answer. + +Context: +{context} + +Question: +{question} + +Answer: +""" + +rag_prompt_template = PromptTemplate.from_template(template) diff --git a/backend/app/services/rag_chain_service.py b/backend/app/services/rag_chain_service.py new file mode 100644 index 0000000..27412f2 --- /dev/null +++ b/backend/app/services/rag_chain_service.py @@ -0,0 +1,58 @@ +from functools import lru_cache + +from langchain_core.output_parsers import StrOutputParser +from langchain_core.runnables import RunnableParallel, RunnablePassthrough + +from app.core.config import get_settings +from app.services.prompt_service import rag_prompt_template +from app.services.retriever_service import get_retriever_service + + +def _get_llm(): + """Use OpenAI if OPENAI_API_KEY is set, otherwise Ollama.""" + settings = get_settings() + if settings.llm_provider == "openai": + from langchain_openai import ChatOpenAI + return ChatOpenAI( + model=settings.openai_model, + api_key=settings.openai_api_key, + temperature=0.4, + ) + from langchain_ollama import OllamaLLM + return OllamaLLM( + model=settings.ollama_model, + base_url=settings.ollama_base_url, + ) + + +class RagChainService: + def __init__(self): + self.retriever = get_retriever_service().get_retriever() + self.llm = _get_llm() + self._chain = None + + def get_chain(self): + if self._chain is not None: + return self._chain + + def format_docs(docs): + return "\n\n".join(doc.page_content for doc in docs) + + rag_chain_from_docs = ( + RunnablePassthrough.assign(context=(lambda x: format_docs(x["context"]))) + | rag_prompt_template + | self.llm + | StrOutputParser() + ) + + rag_chain_with_source = RunnableParallel( + {"context": self.retriever, "question": RunnablePassthrough()} + ).assign(answer=rag_chain_from_docs) + + self._chain = rag_chain_with_source + return self._chain + + +@lru_cache +def get_rag_chain_service() -> RagChainService: + return RagChainService() diff --git a/backend/app/services/rag_service.py b/backend/app/services/rag_service.py new file mode 100644 index 0000000..d231eb7 --- /dev/null +++ b/backend/app/services/rag_service.py @@ -0,0 +1,23 @@ +import asyncio +from app.services.llm_service import LLMService +async def generate_answer(query: str) -> str: + await asyncio.sleep(1) + return f"Answer: {query}" + +class RAGService: + def __init__(self): + pass + + async def answer(self, query: str) -> str: + llm_service = LLMService() + # print(f"Test: {llm_service.call()}") + # answer = await generate_answer(query=query) + answer = await llm_service.a_call() + return answer + + async def test_llm(self, query: str) -> str: + llm_service = LLMService() + answer = await llm_service.a_call() + return answer + + \ No newline at end of file diff --git a/backend/app/services/retriever_service.py b/backend/app/services/retriever_service.py new file mode 100644 index 0000000..18b4c4b --- /dev/null +++ b/backend/app/services/retriever_service.py @@ -0,0 +1,85 @@ +""" +Chroma-backed retriever. Built on first use and cached, so importing this module +never touches Chroma, OpenAI or Ollama. + +get_embeddings() is shared by the retriever and scripts/seed_catalog.py: vectors +are only comparable when the same provider and model produced them, so switching +between OpenAI and Ollama requires re-seeding. +""" + +import logging +from functools import lru_cache + +import chromadb +from langchain_chroma import Chroma + +from app.core.config import Settings, get_settings + +logger = logging.getLogger(__name__) + +# Collection metadata keys written by the seed script. +EMBEDDING_PROVIDER_KEY = "embedding_provider" +EMBEDDING_MODEL_KEY = "embedding_model" + + +def get_embeddings(settings: Settings | None = None): + """Use OpenAI embeddings if OPENAI_API_KEY is set, otherwise Ollama.""" + settings = settings or get_settings() + if settings.llm_provider == "openai": + from langchain_openai import OpenAIEmbeddings + return OpenAIEmbeddings( + model=settings.openai_embedding_model, + api_key=settings.openai_api_key, + ) + from langchain_ollama import OllamaEmbeddings + return OllamaEmbeddings( + model=settings.ollama_embedding_model, + base_url=settings.ollama_base_url, + ) + + +def embedding_signature(settings: Settings | None = None) -> dict: + settings = settings or get_settings() + return { + EMBEDDING_PROVIDER_KEY: settings.llm_provider, + EMBEDDING_MODEL_KEY: settings.embedding_model, + } + + +class RetrieverService: + def __init__(self, settings: Settings | None = None): + settings = settings or get_settings() + self.client = chromadb.PersistentClient(path=settings.chroma_persist_directory) + self.collection_name = settings.chroma_collection_name + self.embeddings = get_embeddings(settings) + self._warn_on_embedding_mismatch(settings) + + def _warn_on_embedding_mismatch(self, settings: Settings): + try: + collection = self.client.get_collection(self.collection_name) + except Exception: + logger.warning( + "Chroma collection %r does not exist yet; run scripts/seed_catalog.py", + self.collection_name, + ) + return + seeded = {k: (collection.metadata or {}).get(k) for k in embedding_signature(settings)} + if seeded != embedding_signature(settings): + logger.warning( + "Collection %r was seeded with %s but the API is configured for %s; " + "retrieval will be wrong until you re-run scripts/seed_catalog.py", + self.collection_name, seeded, embedding_signature(settings), + ) + + def get_retriever(self): + vector_store = Chroma( + client=self.client, + collection_name=self.collection_name, + embedding_function=self.embeddings, + ) + return vector_store.as_retriever() + + +@lru_cache +def get_retriever_service() -> RetrieverService: + return RetrieverService() diff --git a/backend/app/services/scraper_service.py b/backend/app/services/scraper_service.py new file mode 100644 index 0000000..70dc8dc --- /dev/null +++ b/backend/app/services/scraper_service.py @@ -0,0 +1,305 @@ +""" +UTD Catalog Scraper +Scrapes program requirements and example degree plans from UTD undergraduate catalog. +""" + +import asyncio +import logging +import re +from pathlib import Path +from typing import List, Optional, Tuple +from urllib.parse import urljoin + +from playwright.async_api import async_playwright, Browser, Page, TimeoutError as PlaywrightTimeoutError + +logging.basicConfig( + level=logging.WARNING, + format='%(asctime)s - %(levelname)s - %(message)s' +) +logger = logging.getLogger(__name__) + + +class UTDCatalogScraper: + """Scraper for UTD undergraduate catalog program pages.""" + + BASE_URL = "https://catalog.utdallas.edu/2025/undergraduate/programs" + + def __init__( + self, + max_pages: Optional[int] = None, + rate_limit: float = 1.0, + max_parallel: int = 3, + output_dir: str = "./output" + ): + """ + Initialize the scraper. + + Args: + max_pages: Maximum number of pages to scrape (None for all) + rate_limit: Delay between requests in seconds + max_parallel: Maximum number of concurrent browser instances + output_dir: Directory to save scraped data + """ + self.max_pages = max_pages + self.rate_limit = rate_limit + self.max_parallel = max_parallel + self.output_dir = Path(output_dir) + self.output_dir.mkdir(parents=True, exist_ok=True) + self.semaphore = asyncio.Semaphore(max_parallel) + self._last_request_time = 0 + + async def _rate_limit(self): + """Apply rate limiting between requests.""" + current_time = asyncio.get_event_loop().time() + time_since_last = current_time - self._last_request_time + if time_since_last < self.rate_limit: + await asyncio.sleep(self.rate_limit - time_since_last) + self._last_request_time = asyncio.get_event_loop().time() + + def _sanitize_filename(self, name: str) -> str: + """Convert a name to a safe filename.""" + # Remove special characters and replace spaces with underscores + name = re.sub(r'[^\w\s-]', '', name) + name = re.sub(r'[-\s]+', '_', name) + return name.strip('_').lower() + + async def find_program_links(self, page: Page) -> List[Tuple[str, str]]: + """ + Find all program links matching the criteria. + + Returns: + List of tuples (url, program_name) + """ + await page.goto(self.BASE_URL, wait_until="networkidle") + await self._rate_limit() + + links = [] + seen_urls = set() + + # Find all

tags that contain "credit hours" (case-insensitive) + paragraphs = await page.query_selector_all('p') + for p in paragraphs: + p_text = await p.inner_text() + if 'credit hours' in p_text.lower(): + # Find all tags within this paragraph + p_links = await p.query_selector_all('a') + for link in p_links: + href = await link.get_attribute('href') + text = await link.inner_text() + if href: + full_url = urljoin(self.BASE_URL, href) + if full_url not in seen_urls: + seen_urls.add(full_url) + links.append((full_url, text.strip())) + + # Find tags containing "Concentration" (case-insensitive) + all_links = await page.query_selector_all('a') + for link in all_links: + text = await link.inner_text() + if 'concentration' in text.lower(): + href = await link.get_attribute('href') + if href: + full_url = urljoin(self.BASE_URL, href) + if full_url not in seen_urls: + seen_urls.add(full_url) + links.append((full_url, text.strip())) + + logger.info(f"Found {len(links)} program links") + return links + + async def scrape_program_page(self, page: Page, url: str) -> Tuple[Optional[str], Optional[str]]: + """ + Scrape a program page to extract requirements and find example link. + + Returns: + Tuple of (requirements_text, example_url) + """ + try: + await page.goto(url, wait_until="networkidle", timeout=30000) + await self._rate_limit() + + # Extract requirements text (body content, excluding nav/header/footer) + # Try to get main content area + main_content = await page.query_selector('main, .main-content, #content, .content') + if main_content: + requirements_text = await main_content.inner_text() + else: + # Fallback to body but exclude common nav/header/footer elements + body = await page.query_selector('body') + if body: + # Remove script and style elements + await page.evaluate(""" + () => { + const scripts = document.querySelectorAll('script, style, nav, header, footer'); + scripts.forEach(el => el.remove()); + } + """) + requirements_text = await body.inner_text() + else: + requirements_text = await page.inner_text('body') + + # Find example link containing "example" and "degree requirements" (case-insensitive) + example_url = None + example_links = await page.query_selector_all('a') + for link in example_links: + text = await link.inner_text() + href = await link.get_attribute('href') + if text and href: + text_lower = text.lower() + if 'example' in text_lower and 'degree requirements' in text_lower: + example_url = urljoin(url, href) + break + + return requirements_text, example_url + + except PlaywrightTimeoutError: + logger.error(f"Timeout loading page: {url}") + return None, None + except Exception as e: + logger.error(f"Error scraping program page {url}: {e}") + return None, None + + async def scrape_example_page(self, page: Page, url: str) -> Optional[str]: + """ + Scrape an example degree requirements page. + + Returns: + Text content of the example page + """ + try: + await page.goto(url, wait_until="networkidle", timeout=30000) + await self._rate_limit() + + # Extract text content + main_content = await page.query_selector('main, .main-content, #content, .content') + if main_content: + example_text = await main_content.inner_text() + else: + body = await page.query_selector('body') + if body: + await page.evaluate(""" + () => { + const scripts = document.querySelectorAll('script, style, nav, header, footer'); + scripts.forEach(el => el.remove()); + } + """) + example_text = await body.inner_text() + else: + example_text = await page.inner_text('body') + + return example_text + + except PlaywrightTimeoutError: + logger.error(f"Timeout loading example page: {url}") + return None + except Exception as e: + logger.error(f"Error scraping example page {url}: {e}") + return None + + def save_program_data(self, major_name: str, requirements: Optional[str], example: Optional[str]): + """ + Save program data to files. + + Args: + major_name: Name of the major (will be sanitized for folder name) + requirements: Requirements text content + example: Example degree requirements text content + """ + safe_name = self._sanitize_filename(major_name) + major_dir = self.output_dir / safe_name + major_dir.mkdir(parents=True, exist_ok=True) + + # Save requirements + if requirements: + requirements_file = major_dir / "requirements.txt" + requirements_file.write_text(requirements, encoding='utf-8') + logger.info(f"Saved requirements to {requirements_file}") + else: + logger.warning(f"No requirements found for {major_name}") + + # Save example + if example: + example_file = major_dir / "example.txt" + example_file.write_text(example, encoding='utf-8') + logger.info(f"Saved example to {example_file}") + else: + logger.warning(f"No example found for {major_name}") + + async def _scrape_single_program(self, browser: Browser, url: str, name: str): + """Scrape a single program page with semaphore control.""" + async with self.semaphore: + try: + context = await browser.new_context() + page = await context.new_page() + + # Scrape program page + requirements, example_url = await self.scrape_program_page(page, url) + + # Scrape example page if found + example = None + if example_url: + example = await self.scrape_example_page(page, example_url) + + # Save data + self.save_program_data(name, requirements, example) + + await context.close() + + except Exception as e: + logger.error(f"Error scraping {name} ({url}): {e}") + + async def scrape(self): + """Main scraping method.""" + async with async_playwright() as p: + browser = await p.chromium.launch(headless=True) + + try: + # Create a page to find all program links + page = await browser.new_page() + program_links = await self.find_program_links(page) + await page.close() + + # Limit pages if specified + if self.max_pages: + program_links = program_links[:self.max_pages] + + logger.info(f"Scraping {len(program_links)} program pages...") + + # Scrape all programs in parallel (with semaphore limit) + tasks = [ + self._scrape_single_program(browser, url, name) + for url, name in program_links + ] + await asyncio.gather(*tasks, return_exceptions=True) + + finally: + await browser.close() + + logger.info("Scraping completed!") + +""" +Test script to verify scraper works with a single page. +""" + + +import sys +from pathlib import Path + +# Add parent directory to path to allow importing scraper +sys.path.insert(0, str(Path(__file__).parent.parent)) + + +async def main(): + scraper = UTDCatalogScraper( + max_pages=1, + rate_limit=2.0, + max_parallel=5, + output_dir="./data" + ) + + await scraper.scrape() + print("Test completed! Check ./data for results.") + + +if __name__ == "__main__": + asyncio.run(main()) \ No newline at end of file diff --git a/backend/app/services/vector_service.py b/backend/app/services/vector_service.py new file mode 100644 index 0000000..724caa5 --- /dev/null +++ b/backend/app/services/vector_service.py @@ -0,0 +1,9 @@ +class VectorService: + def __init__(self): + pass + + async def embed(self): + pass + + async def search(self): + pass \ No newline at end of file diff --git a/backend/pytest.ini b/backend/pytest.ini new file mode 100644 index 0000000..eeb9d41 --- /dev/null +++ b/backend/pytest.ini @@ -0,0 +1,3 @@ +[pytest] +testpaths = tests +addopts = -q diff --git a/backend/requirements-dev.txt b/backend/requirements-dev.txt new file mode 100644 index 0000000..e28596e --- /dev/null +++ b/backend/requirements-dev.txt @@ -0,0 +1,4 @@ +# Test tooling. Install on top of requirements.txt. +-r requirements.txt +pytest>=8,<10 +httpx>=0.27 diff --git a/backend/requirements-scrape.txt b/backend/requirements-scrape.txt new file mode 100644 index 0000000..60b8d5f --- /dev/null +++ b/backend/requirements-scrape.txt @@ -0,0 +1,11 @@ +# Scraping and heavy ingestion tooling. Not needed to run the API or seed the +# vector store. Install on top of requirements.txt: +# pip install -r requirements.txt -r requirements-scrape.txt +# playwright install chromium +playwright>=1.48.0 +Crawl4AI==0.7.6 +docling==2.60.0 +docling-core==2.50.0 +transformers==4.57.1 +torch==2.9.0 +sentence-transformers diff --git a/backend/requirements.txt b/backend/requirements.txt index e7639e8..e77475f 100644 --- a/backend/requirements.txt +++ b/backend/requirements.txt @@ -1 +1,18 @@ -playwright>=1.48.0 \ No newline at end of file +# Runtime dependencies for the AskTemoc API (also installed by the Dockerfile). +# Heavy scraping/ingestion tooling lives in requirements-scrape.txt; test tooling in +# requirements-dev.txt. +fastapi==0.142.2 +uvicorn[standard]==0.54.0 +SQLAlchemy==2.1.3 +pydantic==2.13.5 +pydantic-settings==2.15.0 + +# RAG / LLM stack +langchain-core==1.6.6 +langchain-community==0.4.2 +langchain-text-splitters==1.1.3 +langchain-chroma==1.1.0 +langchain-ollama==1.1.0 +langchain-openai==1.6.7 +chromadb==1.5.9 +openai==3.24.0 diff --git a/backend/scrapers/catalog/README.md b/backend/scrapers/catalog/README.md new file mode 100644 index 0000000..86d9d04 --- /dev/null +++ b/backend/scrapers/catalog/README.md @@ -0,0 +1,93 @@ +# UTD Catalog Scraper + +A Python web scraper using Playwright to extract program requirements and example +degree plans from the UTD undergraduate catalog. It produces the plain-text corpus +the assistant answers from. + +> The committed corpus lives in `data/catalog/` at the repo root, and +> `backend/scripts/seed_catalog.py` loads it into the vector store. Rerun the +> scraper only to refresh that snapshot. + +## Installation + +1. Create a virtual environment and install dependencies: + +```bash +cd backend +python3 -m venv .venv +source .venv/bin/activate +pip install -r requirements.txt -r requirements-scrape.txt +``` + +2. Install the browser build matching the installed Playwright version: + +```bash +playwright install chromium +``` + +Run this from the activated venv. Playwright pins an exact browser build, so a +Chromium installed by a different Playwright version won't be used and launching +fails with `Executable doesn't exist`. + +## Usage + +```python +import asyncio +from scraper import UTDCatalogScraper + +asyncio.run(UTDCatalogScraper().scrape()) +``` + +### With Configuration + +```python +scraper = UTDCatalogScraper( + max_pages=10, # Limit to 10 programs + rate_limit=2.0, # 2 second delay between requests + max_parallel=5, # 5 concurrent browser contexts + output_dir="./data" # Custom output directory +) +asyncio.run(scraper.scrape()) +``` + +### Full run + +```bash +python test_scraper.py +``` + +Run from `backend/scrapers/catalog/`. Despite the name, this scrapes **every** +program (`max_pages=None`) into `./test_output` (gitignored) with `rate_limit=2.0` and `max_parallel=5`. Expect roughly 115 +programs and several minutes. Pass `max_pages` to try a handful first. + +## Configuration Options + +- `max_pages`: Limit number of programs to scrape (None for all) +- `rate_limit`: Delay between requests in seconds (default: 1.0) +- `max_parallel`: Number of concurrent browser contexts (default: 3) +- `output_dir`: Output directory path (default: `./output`) + +## How Programs Are Discovered + +From the catalog's program index, the scraper collects links inside any `

` +mentioning "credit hours", plus any link whose text mentions "Concentration". +Footnote anchors (`…/programs#footnote-1`, link text `1`) and the index page itself +are skipped. URL fragments are kept, since concentrations of the same degree differ +only by fragment. + +## Output Structure + +``` +output/ + {major_name}/ + requirements.txt program page text + example.txt example degree plan, when the program links to one +``` + +`example.txt` is absent for programs with no "example degree requirements" link — +most double majors, for instance. Extracted text currently includes some page +chrome (for example "Download page as a PDF"), because the catalog's markup doesn't +match the `main`/`#content` selectors the scraper prefers. + +`data/catalog/` at the repo root is a committed snapshot of a previous full run. +To refresh it, copy a new `test_output/` over `data/catalog/` and re-seed. diff --git a/backend/__init__.py b/backend/scrapers/catalog/__init__.py similarity index 100% rename from backend/__init__.py rename to backend/scrapers/catalog/__init__.py diff --git a/backend/scrapers/catalog/build_sources.py b/backend/scrapers/catalog/build_sources.py new file mode 100644 index 0000000..9ef4d1f --- /dev/null +++ b/backend/scrapers/catalog/build_sources.py @@ -0,0 +1,58 @@ +""" +Write data/catalog/sources.json: program folder name -> catalog URL and title. + +Reuses the scraper's own link discovery and folder naming, so the keys match the +folders a full scrape produces. Folders with no discovered link are left out; +seeding falls back to the catalog's programs index URL for those. + +Run from backend/scrapers/catalog/ after `playwright install chromium`: + python build_sources.py +""" + +import asyncio +import json +from pathlib import Path + +from playwright.async_api import async_playwright + +from scraper import UTDCatalogScraper + +CATALOG_DIR = Path(__file__).resolve().parents[3] / "data" / "catalog" + + +async def main(): + scraper = UTDCatalogScraper(output_dir=str(Path(__file__).parent / "output")) + async with async_playwright() as p: + browser = await p.chromium.launch(headless=True) + try: + page = await browser.new_page() + links = await scraper.find_program_links(page) + finally: + await browser.close() + + existing = {d.name for d in CATALOG_DIR.iterdir() if d.is_dir()} + sources = {} + collisions = [] + for url, name in links: + folder = scraper._sanitize_filename(name) + if folder not in existing: + continue + # Links with the same name share one folder in a scrape (whichever page + # finished last wins), so the URL is ambiguous; keep the first, report it. + if folder in sources: + collisions.append(folder) + continue + sources[folder] = {"url": url, "title": " ".join(name.split())} + + out = CATALOG_DIR / "sources.json" + out.write_text(json.dumps(dict(sorted(sources.items())), indent=2) + "\n", encoding="utf-8") + missing = sorted(existing - sources.keys()) + print(f"Discovered {len(links)} links; mapped {len(sources)}/{len(existing)} folders -> {out}") + if missing: + print("No link found for:", ", ".join(missing)) + if collisions: + print("Ambiguous (several links share a name):", ", ".join(sorted(set(collisions)))) + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/backend/scraper.py b/backend/scrapers/catalog/scraper.py similarity index 88% rename from backend/scraper.py rename to backend/scrapers/catalog/scraper.py index 34b5467..4ed0d1c 100644 --- a/backend/scraper.py +++ b/backend/scrapers/catalog/scraper.py @@ -63,6 +63,27 @@ def _sanitize_filename(self, name: str) -> str: name = re.sub(r'[-\s]+', '_', name) return name.strip('_').lower() + def _program_url(self, href: Optional[str], text: str) -> Optional[str]: + """ + Resolve a program href to an absolute URL, or None if it isn't a program. + + Filters out footnote anchors (e.g. `programs#footnote-1`, whose link text is + just "1"), which otherwise get scraped as a program named "1". Fragments are + preserved because concentrations are distinguished only by their fragment. + """ + if not href or not text.strip(): + return None + # Numeric link text is a footnote marker, never a program name. + if text.strip().isdigit(): + return None + full = urljoin(self.BASE_URL, href) + path, _, fragment = full.partition('#') + if fragment.startswith('footnote'): + return None + if not fragment and path.rstrip('/') == self.BASE_URL.rstrip('/'): + return None + return full + async def find_program_links(self, page: Page) -> List[Tuple[str, str]]: """ Find all program links matching the criteria. @@ -86,11 +107,10 @@ async def find_program_links(self, page: Page) -> List[Tuple[str, str]]: for link in p_links: href = await link.get_attribute('href') text = await link.inner_text() - if href: - full_url = urljoin(self.BASE_URL, href) - if full_url not in seen_urls: - seen_urls.add(full_url) - links.append((full_url, text.strip())) + full_url = self._program_url(href, text) + if full_url and full_url not in seen_urls: + seen_urls.add(full_url) + links.append((full_url, text.strip())) # Find tags containing "Concentration" (case-insensitive) all_links = await page.query_selector_all('a') @@ -98,11 +118,10 @@ async def find_program_links(self, page: Page) -> List[Tuple[str, str]]: text = await link.inner_text() if 'concentration' in text.lower(): href = await link.get_attribute('href') - if href: - full_url = urljoin(self.BASE_URL, href) - if full_url not in seen_urls: - seen_urls.add(full_url) - links.append((full_url, text.strip())) + full_url = self._program_url(href, text) + if full_url and full_url not in seen_urls: + seen_urls.add(full_url) + links.append((full_url, text.strip())) logger.info(f"Found {len(links)} program links") return links @@ -206,6 +225,9 @@ def save_program_data(self, major_name: str, requirements: Optional[str], exampl example: Example degree requirements text content """ safe_name = self._sanitize_filename(major_name) + if not safe_name: + logger.warning(f"Skipping program with unusable name: {major_name!r}") + return major_dir = self.output_dir / safe_name major_dir.mkdir(parents=True, exist_ok=True) diff --git a/backend/test_scraper.py b/backend/scrapers/catalog/test_scraper.py similarity index 100% rename from backend/test_scraper.py rename to backend/scrapers/catalog/test_scraper.py diff --git a/backend/scripts/seed_catalog.py b/backend/scripts/seed_catalog.py new file mode 100644 index 0000000..7c45c40 --- /dev/null +++ b/backend/scripts/seed_catalog.py @@ -0,0 +1,143 @@ +""" +Seed the Chroma collection the retriever reads from with the committed catalog +snapshot in data/catalog/*/{requirements,example}.txt. + +Idempotent: every chunk previously written by this script is removed before the +catalog is re-added, so re-running replaces rather than duplicates. Uses the same +embeddings as the API (OpenAI if OPENAI_API_KEY is set, else Ollama); if the +collection was seeded with a different provider/model it is dropped and rebuilt, +since vectors from different models are not comparable. + + cd backend && python scripts/seed_catalog.py [--data-dir PATH] +""" + +import argparse +import json +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +import chromadb # noqa: E402 +from langchain_chroma import Chroma # noqa: E402 +from langchain_core.documents import Document # noqa: E402 + +from app.core.config import REPO_ROOT, get_settings # noqa: E402 +from app.services.data_ingestion_service import make_text_splitter # noqa: E402 +from app.services.retriever_service import embedding_signature, get_embeddings # noqa: E402 + +CATALOG_INDEX_URL = "https://catalog.utdallas.edu/2025/undergraduate/programs" +FILES = { + "requirements.txt": "requirements", + "example.txt": "example degree plan", +} +CORPUS = "catalog" +BATCH_SIZE = 64 + + +def load_documents(data_dir: Path) -> list[Document]: + sources_file = data_dir / "sources.json" + sources = json.loads(sources_file.read_text(encoding="utf-8")) if sources_file.exists() else {} + + docs = [] + for program_dir in sorted(p for p in data_dir.iterdir() if p.is_dir()): + program = program_dir.name + info = sources.get(program, {}) + title = info.get("title") or program.replace("_", " ").title() + for filename, kind in FILES.items(): + path = program_dir / filename + if not path.exists(): + continue + text = path.read_text(encoding="utf-8").strip() + if not text: + continue + try: + rel = path.relative_to(REPO_ROOT).as_posix() + except ValueError: + rel = path.as_posix() + docs.append(Document( + page_content=text, + metadata={ + "source": info.get("url", CATALOG_INDEX_URL), + "title": title if kind == "requirements" else f"{title} ({kind})", + "program": program, + "file_path": rel, + "doc_type": kind, + "corpus": CORPUS, + }, + )) + return docs + + +def split(docs: list[Document]) -> tuple[list[Document], list[str]]: + splitter = make_text_splitter() + chunks, ids = [], [] + for doc in docs: + for i, chunk in enumerate(splitter.split_documents([doc])): + chunk.metadata["chunk_index"] = i + chunks.append(chunk) + ids.append(f"{CORPUS}:{doc.metadata['file_path']}#{i}") + return chunks, ids + + +def get_collection(client, name: str, signature: dict): + try: + existing = client.get_collection(name) + except Exception: + existing = None + if existing is not None: + seeded = {k: (existing.metadata or {}).get(k) for k in signature} + if seeded != signature: + print(f"Collection {name!r} was built with {seeded}; rebuilding for {signature}.") + client.delete_collection(name) + return client.get_or_create_collection(name, metadata=signature) + + +def main(): + settings = get_settings() + parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) + parser.add_argument("--data-dir", default=settings.catalog_data_dir) + parser.add_argument("--batch-size", type=int, default=BATCH_SIZE, + help="chunks per embedding request (lower it if a local Ollama struggles)") + args = parser.parse_args() + data_dir = Path(args.data_dir).resolve() + if not data_dir.is_dir(): + sys.exit(f"Catalog directory not found: {data_dir}") + + docs = load_documents(data_dir) + chunks, ids = split(docs) + programs = len({d.metadata["program"] for d in docs}) + print(f"Loaded {len(docs)} documents from {programs} programs in {data_dir}") + print(f"Split into {len(chunks)} chunks") + + signature = embedding_signature(settings) + print(f"Embeddings: {signature['embedding_provider']} / {signature['embedding_model']}") + client = chromadb.PersistentClient(path=settings.chroma_persist_directory) + collection = get_collection(client, settings.chroma_collection_name, signature) + collection.delete(where={"corpus": CORPUS}) + + store = Chroma( + client=client, + collection_name=settings.chroma_collection_name, + embedding_function=get_embeddings(settings), + ) + batch = args.batch_size + for start in range(0, len(chunks), batch): + try: + store.add_documents(chunks[start:start + batch], ids=ids[start:start + batch]) + except Exception as e: + sys.exit( + f"Embedding failed after {start} of {len(chunks)} chunks: {e}\n" + "Check OPENAI_API_KEY (and its billing) or that Ollama is running with " + f"{settings.ollama_embedding_model!r} pulled, then re-run; re-running is safe." + ) + print(f" embedded {min(start + batch, len(chunks))}/{len(chunks)}") + + print( + f"Done: collection {settings.chroma_collection_name!r} at " + f"{settings.chroma_persist_directory} now holds {collection.count()} chunks." + ) + + +if __name__ == "__main__": + main() diff --git a/backend/tests/__init__.py b/backend/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/backend/tests/conftest.py b/backend/tests/conftest.py new file mode 100644 index 0000000..fb8e8b9 --- /dev/null +++ b/backend/tests/conftest.py @@ -0,0 +1,75 @@ +""" +Test setup. Every setting the app reads is pinned here, before the app is +imported, so a developer's backend/.env (API keys, ...) never leaks in. Nothing +in the suite may reach OpenAI, Ollama or the network: the RAG chain is replaced +by a fake and Chroma is never opened. +""" + +import os +import sys +import tempfile +from pathlib import Path + +import pytest + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +_tmp = tempfile.mkdtemp(prefix="asktemoc-tests-") + +os.environ.update({ + "ENVIRONMENT": "development", + "DATABASE_URL": f"sqlite:///{Path(_tmp, 'test.db').as_posix()}", + "CHROMA_PERSIST_DIRECTORY": str(Path(_tmp, "chroma")), + "OPENAI_API_KEY": "", + "OLLAMA_BASE_URL": "http://ollama.invalid:11434", + "CORS_ORIGINS": "http://localhost:3000", +}) + +from fastapi.testclient import TestClient # noqa: E402 +from langchain_core.documents import Document # noqa: E402 + +from app.api.endpoints.rag_endpoint import get_chain # noqa: E402 +from app.core.config import get_settings # noqa: E402 +from app.main import app # noqa: E402 + + +class FakeChain: + """Mimics the RAG chain's astream output: context first, then answer deltas.""" + + def __init__(self, deltas=("Core ", "courses ", "are CS 1337.")): + self.deltas = deltas + self.calls = [] + + async def astream(self, question): + self.calls.append(question) + yield {"question": question} + yield {"context": [ + Document(page_content="...", metadata={ + "source": "https://catalog.utdallas.edu/2025/undergraduate/programs/ecs/computer-science", + "title": "Bachelor of Science in Computer Science", + }), + Document(page_content="...", metadata={"source": "https://example.edu/other"}), + ]} + for d in self.deltas: + yield {"answer": d} + + +@pytest.fixture +def fake_chain(): + chain = FakeChain() + app.dependency_overrides[get_chain] = lambda: chain + yield chain + app.dependency_overrides.pop(get_chain, None) + + +@pytest.fixture +def client(): + with TestClient(app) as c: + yield c + + +@pytest.fixture(autouse=True) +def _fresh_settings(): + get_settings.cache_clear() + yield + get_settings.cache_clear() diff --git a/backend/tests/test_chat.py b/backend/tests/test_chat.py new file mode 100644 index 0000000..1940a27 --- /dev/null +++ b/backend/tests/test_chat.py @@ -0,0 +1,50 @@ +import json + +from app.api.endpoints.rag_endpoint import get_chain +from app.main import app + + +def _post(client, message="core requirements for BS Computer Science"): + return client.post("/api/chat", json={"message": message}) + + +def _events(body: str): + return [json.loads(line[len("data: "):]) for line in body.splitlines() if line.startswith("data: ")] + + +def test_chat_streams_well_formed_sse(client, fake_chain): + r = _post(client) + assert r.status_code == 200 + assert r.headers["content-type"].startswith("text/event-stream") + events = _events(r.text) + + assert [e["type"] for e in events] == ["source", "source", "text", "text", "text", "done"] + assert events[0]["message"] == ( + "Source 1: https://catalog.utdallas.edu/2025/undergraduate/programs/ecs/computer-science" + ) + assert events[0]["index"] == 1 + assert events[0]["title"] == "Bachelor of Science in Computer Science" + assert events[1]["message"] == "Source 2: https://example.edu/other" + assert "title" not in events[1] + # Deltas, not cumulative prefixes: concatenating them yields the answer once. + assert "".join(e["message"] for e in events if e["type"] == "text") == "Core courses are CS 1337." + assert fake_chain.calls == ["core requirements for BS Computer Science"] + + +def test_chat_requires_a_message(client, fake_chain): + assert client.post("/api/chat", json={}).status_code == 422 + assert fake_chain.calls == [] + + +def test_chain_failure_reports_error_then_done(client): + class Broken: + async def astream(self, question): + raise RuntimeError("LLM down") + yield # makes this an async generator + + app.dependency_overrides[get_chain] = lambda: Broken() + try: + events = _events(_post(client).text) + finally: + app.dependency_overrides.pop(get_chain, None) + assert events == [{"type": "text", "message": "Error processing request"}, {"type": "done"}] diff --git a/backend/tests/test_config.py b/backend/tests/test_config.py new file mode 100644 index 0000000..3bef9c0 --- /dev/null +++ b/backend/tests/test_config.py @@ -0,0 +1,37 @@ +import pytest +from pydantic import ValidationError + +from app.core.config import BACKEND_DIR, Settings + + +def _settings(**overrides): + return Settings(_env_file=None, **overrides) + + +def test_validation_errors_hide_secret_values(): + # A bad value for a secret field must not be echoed into logs. + with pytest.raises(ValidationError) as exc: + _settings(openai_api_key=["sk-secret-value"]) + assert "sk-secret-value" not in str(exc.value) + + +def test_blank_openai_key_selects_ollama(): + s = _settings(openai_api_key="") + assert s.openai_api_key is None + assert s.llm_provider == "ollama" + assert s.embedding_model == s.ollama_embedding_model + + +def test_openai_key_selects_openai(): + s = _settings(openai_api_key="sk-test") + assert s.llm_provider == "openai" + assert s.embedding_model == s.openai_embedding_model + + +def test_csv_and_path_parsing(): + s = _settings( + cors_origins="http://a.test, http://b.test", + database_url="sqlite:///./rel.db", + ) + assert s.cors_origins == ["http://a.test", "http://b.test"] + assert s.database_url == "sqlite:///" + (BACKEND_DIR / "rel.db").as_posix() diff --git a/backend/tests/test_health_and_routes.py b/backend/tests/test_health_and_routes.py new file mode 100644 index 0000000..ea42795 --- /dev/null +++ b/backend/tests/test_health_and_routes.py @@ -0,0 +1,32 @@ +from collections import Counter + +from app.main import app +from app.services.rag_chain_service import get_rag_chain_service +from app.services.retriever_service import get_retriever_service + + +def test_health_with_trailing_slash(client): + r = client.get("/api/health/") + assert r.status_code == 200 + assert r.json() == {"status": "healthy"} + + +def test_health_without_trailing_slash_does_not_redirect(client): + r = client.get("/api/health", follow_redirects=False) + assert r.status_code == 200 + assert r.json() == {"status": "healthy"} + + +def test_no_duplicate_method_and_path(): + seen = Counter( + (route.path, method) + for route in app.routes + for method in (getattr(route, "methods", None) or ()) + ) + assert [k for k, n in seen.items() if n > 1] == [] + + +def test_import_and_health_do_not_build_rag_services(client): + client.get("/api/health/") + assert get_rag_chain_service.cache_info().currsize == 0 + assert get_retriever_service.cache_info().currsize == 0 diff --git a/backend/tests/test_seed_catalog.py b/backend/tests/test_seed_catalog.py new file mode 100644 index 0000000..581343e --- /dev/null +++ b/backend/tests/test_seed_catalog.py @@ -0,0 +1,47 @@ +import importlib.util +import json +from pathlib import Path + +_spec = importlib.util.spec_from_file_location( + "seed_catalog", Path(__file__).resolve().parents[1] / "scripts" / "seed_catalog.py" +) +seed = importlib.util.module_from_spec(_spec) +_spec.loader.exec_module(seed) + + +def _catalog(tmp_path): + cs = tmp_path / "bachelor_of_science_in_computer_science" + cs.mkdir() + (cs / "requirements.txt").write_text("CS 1337 Computer Science I\n" * 120, encoding="utf-8") + (cs / "example.txt").write_text("Freshman year: CS 1337", encoding="utf-8") + orphan = tmp_path / "1" + orphan.mkdir() + (orphan / "requirements.txt").write_text("footnote page", encoding="utf-8") + (tmp_path / "sources.json").write_text(json.dumps({ + cs.name: {"url": "https://catalog.utdallas.edu/cs", "title": "Bachelor of Science in Computer Science"}, + }), encoding="utf-8") + return tmp_path + + +def test_load_documents_sets_source_metadata(tmp_path): + docs = seed.load_documents(_catalog(tmp_path)) + meta = {(d.metadata["program"], d.metadata["doc_type"]): d.metadata for d in docs} + + req = meta[("bachelor_of_science_in_computer_science", "requirements")] + assert req["source"] == "https://catalog.utdallas.edu/cs" + assert req["title"] == "Bachelor of Science in Computer Science" + example = meta[("bachelor_of_science_in_computer_science", "example degree plan")] + assert example["title"].endswith("(example degree plan)") + # Folders missing from sources.json fall back to the programs index. + assert meta[("1", "requirements")]["source"] == seed.CATALOG_INDEX_URL + assert all(m["corpus"] == "catalog" for m in meta.values()) + + +def test_split_ids_are_stable_and_unique(tmp_path): + docs = seed.load_documents(_catalog(tmp_path)) + chunks, ids = seed.split(docs) + _, ids_again = seed.split(seed.load_documents(tmp_path)) + assert len(chunks) == len(ids) > len(docs) + assert len(set(ids)) == len(ids) + assert ids == ids_again + assert max(len(c.page_content) for c in chunks) <= 1000 diff --git a/backend/test_output/1/requirements.txt b/data/catalog/1/requirements.txt similarity index 100% rename from backend/test_output/1/requirements.txt rename to data/catalog/1/requirements.txt diff --git a/backend/test_output/2/requirements.txt b/data/catalog/2/requirements.txt similarity index 100% rename from backend/test_output/2/requirements.txt rename to data/catalog/2/requirements.txt diff --git a/backend/test_output/3/requirements.txt b/data/catalog/3/requirements.txt similarity index 100% rename from backend/test_output/3/requirements.txt rename to data/catalog/3/requirements.txt diff --git a/backend/test_output/ba_in_biology_and_criminology_double_major/requirements.txt b/data/catalog/ba_in_biology_and_criminology_double_major/requirements.txt similarity index 100% rename from backend/test_output/ba_in_biology_and_criminology_double_major/requirements.txt rename to data/catalog/ba_in_biology_and_criminology_double_major/requirements.txt diff --git a/backend/test_output/ba_in_criminology_and_biology_double_major/requirements.txt b/data/catalog/ba_in_criminology_and_biology_double_major/requirements.txt similarity index 100% rename from backend/test_output/ba_in_criminology_and_biology_double_major/requirements.txt rename to data/catalog/ba_in_criminology_and_biology_double_major/requirements.txt diff --git a/backend/test_output/ba_in_education_7_12_social_studies_with_a_concentration_in_economics/example.txt b/data/catalog/ba_in_education_7_12_social_studies_with_a_concentration_in_economics/example.txt similarity index 100% rename from backend/test_output/ba_in_education_7_12_social_studies_with_a_concentration_in_economics/example.txt rename to data/catalog/ba_in_education_7_12_social_studies_with_a_concentration_in_economics/example.txt diff --git a/backend/test_output/ba_in_education_7_12_social_studies_with_a_concentration_in_economics/requirements.txt b/data/catalog/ba_in_education_7_12_social_studies_with_a_concentration_in_economics/requirements.txt similarity index 100% rename from backend/test_output/ba_in_education_7_12_social_studies_with_a_concentration_in_economics/requirements.txt rename to data/catalog/ba_in_education_7_12_social_studies_with_a_concentration_in_economics/requirements.txt diff --git a/backend/test_output/ba_in_education_7_12_social_studies_with_a_concentration_in_geography/example.txt b/data/catalog/ba_in_education_7_12_social_studies_with_a_concentration_in_geography/example.txt similarity index 100% rename from backend/test_output/ba_in_education_7_12_social_studies_with_a_concentration_in_geography/example.txt rename to data/catalog/ba_in_education_7_12_social_studies_with_a_concentration_in_geography/example.txt diff --git a/backend/test_output/ba_in_education_7_12_social_studies_with_a_concentration_in_geography/requirements.txt b/data/catalog/ba_in_education_7_12_social_studies_with_a_concentration_in_geography/requirements.txt similarity index 100% rename from backend/test_output/ba_in_education_7_12_social_studies_with_a_concentration_in_geography/requirements.txt rename to data/catalog/ba_in_education_7_12_social_studies_with_a_concentration_in_geography/requirements.txt diff --git a/backend/test_output/ba_in_education_7_12_social_studies_with_a_concentration_in_government/example.txt b/data/catalog/ba_in_education_7_12_social_studies_with_a_concentration_in_government/example.txt similarity index 100% rename from backend/test_output/ba_in_education_7_12_social_studies_with_a_concentration_in_government/example.txt rename to data/catalog/ba_in_education_7_12_social_studies_with_a_concentration_in_government/example.txt diff --git a/backend/test_output/ba_in_education_7_12_social_studies_with_a_concentration_in_government/requirements.txt b/data/catalog/ba_in_education_7_12_social_studies_with_a_concentration_in_government/requirements.txt similarity index 100% rename from backend/test_output/ba_in_education_7_12_social_studies_with_a_concentration_in_government/requirements.txt rename to data/catalog/ba_in_education_7_12_social_studies_with_a_concentration_in_government/requirements.txt diff --git a/backend/test_output/ba_in_education_7_12_social_studies_with_a_concentration_in_history/example.txt b/data/catalog/ba_in_education_7_12_social_studies_with_a_concentration_in_history/example.txt similarity index 100% rename from backend/test_output/ba_in_education_7_12_social_studies_with_a_concentration_in_history/example.txt rename to data/catalog/ba_in_education_7_12_social_studies_with_a_concentration_in_history/example.txt diff --git a/backend/test_output/ba_in_education_7_12_social_studies_with_a_concentration_in_history/requirements.txt b/data/catalog/ba_in_education_7_12_social_studies_with_a_concentration_in_history/requirements.txt similarity index 100% rename from backend/test_output/ba_in_education_7_12_social_studies_with_a_concentration_in_history/requirements.txt rename to data/catalog/ba_in_education_7_12_social_studies_with_a_concentration_in_history/requirements.txt diff --git a/backend/test_output/ba_in_literature_creative_writing_concentration/example.txt b/data/catalog/ba_in_literature_creative_writing_concentration/example.txt similarity index 100% rename from backend/test_output/ba_in_literature_creative_writing_concentration/example.txt rename to data/catalog/ba_in_literature_creative_writing_concentration/example.txt diff --git a/backend/test_output/ba_in_literature_creative_writing_concentration/requirements.txt b/data/catalog/ba_in_literature_creative_writing_concentration/requirements.txt similarity index 100% rename from backend/test_output/ba_in_literature_creative_writing_concentration/requirements.txt rename to data/catalog/ba_in_literature_creative_writing_concentration/requirements.txt diff --git a/backend/test_output/ba_in_literature_pre_law_concentration/example.txt b/data/catalog/ba_in_literature_pre_law_concentration/example.txt similarity index 100% rename from backend/test_output/ba_in_literature_pre_law_concentration/example.txt rename to data/catalog/ba_in_literature_pre_law_concentration/example.txt diff --git a/backend/test_output/ba_in_literature_pre_law_concentration/requirements.txt b/data/catalog/ba_in_literature_pre_law_concentration/requirements.txt similarity index 100% rename from backend/test_output/ba_in_literature_pre_law_concentration/requirements.txt rename to data/catalog/ba_in_literature_pre_law_concentration/requirements.txt diff --git a/backend/test_output/ba_in_literature_professional_and_technical_writing_concentration/example.txt b/data/catalog/ba_in_literature_professional_and_technical_writing_concentration/example.txt similarity index 100% rename from backend/test_output/ba_in_literature_professional_and_technical_writing_concentration/example.txt rename to data/catalog/ba_in_literature_professional_and_technical_writing_concentration/example.txt diff --git a/backend/test_output/ba_in_literature_professional_and_technical_writing_concentration/requirements.txt b/data/catalog/ba_in_literature_professional_and_technical_writing_concentration/requirements.txt similarity index 100% rename from backend/test_output/ba_in_literature_professional_and_technical_writing_concentration/requirements.txt rename to data/catalog/ba_in_literature_professional_and_technical_writing_concentration/requirements.txt diff --git a/backend/test_output/ba_in_literature_spanish_concentration/example.txt b/data/catalog/ba_in_literature_spanish_concentration/example.txt similarity index 100% rename from backend/test_output/ba_in_literature_spanish_concentration/example.txt rename to data/catalog/ba_in_literature_spanish_concentration/example.txt diff --git a/backend/test_output/ba_in_literature_spanish_concentration/requirements.txt b/data/catalog/ba_in_literature_spanish_concentration/requirements.txt similarity index 100% rename from backend/test_output/ba_in_literature_spanish_concentration/requirements.txt rename to data/catalog/ba_in_literature_spanish_concentration/requirements.txt diff --git a/backend/test_output/ba_in_media_arts_and_design_critical_media_studies_concentration/example.txt b/data/catalog/ba_in_media_arts_and_design_critical_media_studies_concentration/example.txt similarity index 100% rename from backend/test_output/ba_in_media_arts_and_design_critical_media_studies_concentration/example.txt rename to data/catalog/ba_in_media_arts_and_design_critical_media_studies_concentration/example.txt diff --git a/backend/test_output/ba_in_media_arts_and_design_critical_media_studies_concentration/requirements.txt b/data/catalog/ba_in_media_arts_and_design_critical_media_studies_concentration/requirements.txt similarity index 100% rename from backend/test_output/ba_in_media_arts_and_design_critical_media_studies_concentration/requirements.txt rename to data/catalog/ba_in_media_arts_and_design_critical_media_studies_concentration/requirements.txt diff --git a/backend/test_output/ba_in_media_arts_and_design_emerging_media_arts_concentration/example.txt b/data/catalog/ba_in_media_arts_and_design_emerging_media_arts_concentration/example.txt similarity index 100% rename from backend/test_output/ba_in_media_arts_and_design_emerging_media_arts_concentration/example.txt rename to data/catalog/ba_in_media_arts_and_design_emerging_media_arts_concentration/example.txt diff --git a/backend/test_output/ba_in_media_arts_and_design_emerging_media_arts_concentration/requirements.txt b/data/catalog/ba_in_media_arts_and_design_emerging_media_arts_concentration/requirements.txt similarity index 100% rename from backend/test_output/ba_in_media_arts_and_design_emerging_media_arts_concentration/requirements.txt rename to data/catalog/ba_in_media_arts_and_design_emerging_media_arts_concentration/requirements.txt diff --git a/backend/test_output/ba_in_visual_and_performing_arts_art_history_concentration/example.txt b/data/catalog/ba_in_visual_and_performing_arts_art_history_concentration/example.txt similarity index 100% rename from backend/test_output/ba_in_visual_and_performing_arts_art_history_concentration/example.txt rename to data/catalog/ba_in_visual_and_performing_arts_art_history_concentration/example.txt diff --git a/backend/test_output/ba_in_visual_and_performing_arts_art_history_concentration/requirements.txt b/data/catalog/ba_in_visual_and_performing_arts_art_history_concentration/requirements.txt similarity index 100% rename from backend/test_output/ba_in_visual_and_performing_arts_art_history_concentration/requirements.txt rename to data/catalog/ba_in_visual_and_performing_arts_art_history_concentration/requirements.txt diff --git a/backend/test_output/ba_in_visual_and_performing_arts_communication_studies_concentration/example.txt b/data/catalog/ba_in_visual_and_performing_arts_communication_studies_concentration/example.txt similarity index 100% rename from backend/test_output/ba_in_visual_and_performing_arts_communication_studies_concentration/example.txt rename to data/catalog/ba_in_visual_and_performing_arts_communication_studies_concentration/example.txt diff --git a/backend/test_output/ba_in_visual_and_performing_arts_communication_studies_concentration/requirements.txt b/data/catalog/ba_in_visual_and_performing_arts_communication_studies_concentration/requirements.txt similarity index 100% rename from backend/test_output/ba_in_visual_and_performing_arts_communication_studies_concentration/requirements.txt rename to data/catalog/ba_in_visual_and_performing_arts_communication_studies_concentration/requirements.txt diff --git a/backend/test_output/ba_in_visual_and_performing_arts_dance_concentration/example.txt b/data/catalog/ba_in_visual_and_performing_arts_dance_concentration/example.txt similarity index 100% rename from backend/test_output/ba_in_visual_and_performing_arts_dance_concentration/example.txt rename to data/catalog/ba_in_visual_and_performing_arts_dance_concentration/example.txt diff --git a/backend/test_output/ba_in_visual_and_performing_arts_dance_concentration/requirements.txt b/data/catalog/ba_in_visual_and_performing_arts_dance_concentration/requirements.txt similarity index 100% rename from backend/test_output/ba_in_visual_and_performing_arts_dance_concentration/requirements.txt rename to data/catalog/ba_in_visual_and_performing_arts_dance_concentration/requirements.txt diff --git a/backend/test_output/ba_in_visual_and_performing_arts_film_concentration/example.txt b/data/catalog/ba_in_visual_and_performing_arts_film_concentration/example.txt similarity index 100% rename from backend/test_output/ba_in_visual_and_performing_arts_film_concentration/example.txt rename to data/catalog/ba_in_visual_and_performing_arts_film_concentration/example.txt diff --git a/backend/test_output/ba_in_visual_and_performing_arts_film_concentration/requirements.txt b/data/catalog/ba_in_visual_and_performing_arts_film_concentration/requirements.txt similarity index 100% rename from backend/test_output/ba_in_visual_and_performing_arts_film_concentration/requirements.txt rename to data/catalog/ba_in_visual_and_performing_arts_film_concentration/requirements.txt diff --git a/backend/test_output/ba_in_visual_and_performing_arts_music_concentration/example.txt b/data/catalog/ba_in_visual_and_performing_arts_music_concentration/example.txt similarity index 100% rename from backend/test_output/ba_in_visual_and_performing_arts_music_concentration/example.txt rename to data/catalog/ba_in_visual_and_performing_arts_music_concentration/example.txt diff --git a/backend/test_output/ba_in_visual_and_performing_arts_music_concentration/requirements.txt b/data/catalog/ba_in_visual_and_performing_arts_music_concentration/requirements.txt similarity index 100% rename from backend/test_output/ba_in_visual_and_performing_arts_music_concentration/requirements.txt rename to data/catalog/ba_in_visual_and_performing_arts_music_concentration/requirements.txt diff --git a/backend/test_output/ba_in_visual_and_performing_arts_photo_video_digital_concentration/example.txt b/data/catalog/ba_in_visual_and_performing_arts_photo_video_digital_concentration/example.txt similarity index 100% rename from backend/test_output/ba_in_visual_and_performing_arts_photo_video_digital_concentration/example.txt rename to data/catalog/ba_in_visual_and_performing_arts_photo_video_digital_concentration/example.txt diff --git a/backend/test_output/ba_in_visual_and_performing_arts_photo_video_digital_concentration/requirements.txt b/data/catalog/ba_in_visual_and_performing_arts_photo_video_digital_concentration/requirements.txt similarity index 100% rename from backend/test_output/ba_in_visual_and_performing_arts_photo_video_digital_concentration/requirements.txt rename to data/catalog/ba_in_visual_and_performing_arts_photo_video_digital_concentration/requirements.txt diff --git a/backend/test_output/ba_in_visual_and_performing_arts_theatre_concentration/example.txt b/data/catalog/ba_in_visual_and_performing_arts_theatre_concentration/example.txt similarity index 100% rename from backend/test_output/ba_in_visual_and_performing_arts_theatre_concentration/example.txt rename to data/catalog/ba_in_visual_and_performing_arts_theatre_concentration/example.txt diff --git a/backend/test_output/ba_in_visual_and_performing_arts_theatre_concentration/requirements.txt b/data/catalog/ba_in_visual_and_performing_arts_theatre_concentration/requirements.txt similarity index 100% rename from backend/test_output/ba_in_visual_and_performing_arts_theatre_concentration/requirements.txt rename to data/catalog/ba_in_visual_and_performing_arts_theatre_concentration/requirements.txt diff --git a/backend/test_output/ba_in_visual_and_performing_arts_visual_arts_concentration/example.txt b/data/catalog/ba_in_visual_and_performing_arts_visual_arts_concentration/example.txt similarity index 100% rename from backend/test_output/ba_in_visual_and_performing_arts_visual_arts_concentration/example.txt rename to data/catalog/ba_in_visual_and_performing_arts_visual_arts_concentration/example.txt diff --git a/backend/test_output/ba_in_visual_and_performing_arts_visual_arts_concentration/requirements.txt b/data/catalog/ba_in_visual_and_performing_arts_visual_arts_concentration/requirements.txt similarity index 100% rename from backend/test_output/ba_in_visual_and_performing_arts_visual_arts_concentration/requirements.txt rename to data/catalog/ba_in_visual_and_performing_arts_visual_arts_concentration/requirements.txt diff --git a/backend/test_output/bachelor_of_arts_in_american_studies/example.txt b/data/catalog/bachelor_of_arts_in_american_studies/example.txt similarity index 100% rename from backend/test_output/bachelor_of_arts_in_american_studies/example.txt rename to data/catalog/bachelor_of_arts_in_american_studies/example.txt diff --git a/backend/test_output/bachelor_of_arts_in_american_studies/requirements.txt b/data/catalog/bachelor_of_arts_in_american_studies/requirements.txt similarity index 100% rename from backend/test_output/bachelor_of_arts_in_american_studies/requirements.txt rename to data/catalog/bachelor_of_arts_in_american_studies/requirements.txt diff --git a/backend/test_output/bachelor_of_arts_in_animation_and_games/example.txt b/data/catalog/bachelor_of_arts_in_animation_and_games/example.txt similarity index 100% rename from backend/test_output/bachelor_of_arts_in_animation_and_games/example.txt rename to data/catalog/bachelor_of_arts_in_animation_and_games/example.txt diff --git a/backend/test_output/bachelor_of_arts_in_animation_and_games/requirements.txt b/data/catalog/bachelor_of_arts_in_animation_and_games/requirements.txt similarity index 100% rename from backend/test_output/bachelor_of_arts_in_animation_and_games/requirements.txt rename to data/catalog/bachelor_of_arts_in_animation_and_games/requirements.txt diff --git a/backend/test_output/bachelor_of_arts_in_biology/example.txt b/data/catalog/bachelor_of_arts_in_biology/example.txt similarity index 100% rename from backend/test_output/bachelor_of_arts_in_biology/example.txt rename to data/catalog/bachelor_of_arts_in_biology/example.txt diff --git a/backend/test_output/bachelor_of_arts_in_biology/requirements.txt b/data/catalog/bachelor_of_arts_in_biology/requirements.txt similarity index 100% rename from backend/test_output/bachelor_of_arts_in_biology/requirements.txt rename to data/catalog/bachelor_of_arts_in_biology/requirements.txt diff --git a/backend/test_output/bachelor_of_arts_in_chemistry/example.txt b/data/catalog/bachelor_of_arts_in_chemistry/example.txt similarity index 100% rename from backend/test_output/bachelor_of_arts_in_chemistry/example.txt rename to data/catalog/bachelor_of_arts_in_chemistry/example.txt diff --git a/backend/test_output/bachelor_of_arts_in_chemistry/requirements.txt b/data/catalog/bachelor_of_arts_in_chemistry/requirements.txt similarity index 100% rename from backend/test_output/bachelor_of_arts_in_chemistry/requirements.txt rename to data/catalog/bachelor_of_arts_in_chemistry/requirements.txt diff --git a/backend/test_output/bachelor_of_arts_in_criminology/example.txt b/data/catalog/bachelor_of_arts_in_criminology/example.txt similarity index 100% rename from backend/test_output/bachelor_of_arts_in_criminology/example.txt rename to data/catalog/bachelor_of_arts_in_criminology/example.txt diff --git a/backend/test_output/bachelor_of_arts_in_criminology/requirements.txt b/data/catalog/bachelor_of_arts_in_criminology/requirements.txt similarity index 100% rename from backend/test_output/bachelor_of_arts_in_criminology/requirements.txt rename to data/catalog/bachelor_of_arts_in_criminology/requirements.txt diff --git a/backend/test_output/bachelor_of_arts_in_economics/example.txt b/data/catalog/bachelor_of_arts_in_economics/example.txt similarity index 100% rename from backend/test_output/bachelor_of_arts_in_economics/example.txt rename to data/catalog/bachelor_of_arts_in_economics/example.txt diff --git a/backend/test_output/bachelor_of_arts_in_economics/requirements.txt b/data/catalog/bachelor_of_arts_in_economics/requirements.txt similarity index 100% rename from backend/test_output/bachelor_of_arts_in_economics/requirements.txt rename to data/catalog/bachelor_of_arts_in_economics/requirements.txt diff --git a/backend/test_output/bachelor_of_arts_in_education/requirements.txt b/data/catalog/bachelor_of_arts_in_education/requirements.txt similarity index 100% rename from backend/test_output/bachelor_of_arts_in_education/requirements.txt rename to data/catalog/bachelor_of_arts_in_education/requirements.txt diff --git a/backend/test_output/bachelor_of_arts_in_history/example.txt b/data/catalog/bachelor_of_arts_in_history/example.txt similarity index 100% rename from backend/test_output/bachelor_of_arts_in_history/example.txt rename to data/catalog/bachelor_of_arts_in_history/example.txt diff --git a/backend/test_output/bachelor_of_arts_in_history/requirements.txt b/data/catalog/bachelor_of_arts_in_history/requirements.txt similarity index 100% rename from backend/test_output/bachelor_of_arts_in_history/requirements.txt rename to data/catalog/bachelor_of_arts_in_history/requirements.txt diff --git a/backend/test_output/bachelor_of_arts_in_interdisciplinary_studies/example.txt b/data/catalog/bachelor_of_arts_in_interdisciplinary_studies/example.txt similarity index 100% rename from backend/test_output/bachelor_of_arts_in_interdisciplinary_studies/example.txt rename to data/catalog/bachelor_of_arts_in_interdisciplinary_studies/example.txt diff --git a/backend/test_output/bachelor_of_arts_in_interdisciplinary_studies/requirements.txt b/data/catalog/bachelor_of_arts_in_interdisciplinary_studies/requirements.txt similarity index 100% rename from backend/test_output/bachelor_of_arts_in_interdisciplinary_studies/requirements.txt rename to data/catalog/bachelor_of_arts_in_interdisciplinary_studies/requirements.txt diff --git a/backend/test_output/bachelor_of_arts_in_international_political_economy/example.txt b/data/catalog/bachelor_of_arts_in_international_political_economy/example.txt similarity index 100% rename from backend/test_output/bachelor_of_arts_in_international_political_economy/example.txt rename to data/catalog/bachelor_of_arts_in_international_political_economy/example.txt diff --git a/backend/test_output/bachelor_of_arts_in_international_political_economy/requirements.txt b/data/catalog/bachelor_of_arts_in_international_political_economy/requirements.txt similarity index 100% rename from backend/test_output/bachelor_of_arts_in_international_political_economy/requirements.txt rename to data/catalog/bachelor_of_arts_in_international_political_economy/requirements.txt diff --git a/backend/test_output/bachelor_of_arts_in_latin_american_studies/example.txt b/data/catalog/bachelor_of_arts_in_latin_american_studies/example.txt similarity index 100% rename from backend/test_output/bachelor_of_arts_in_latin_american_studies/example.txt rename to data/catalog/bachelor_of_arts_in_latin_american_studies/example.txt diff --git a/backend/test_output/bachelor_of_arts_in_latin_american_studies/requirements.txt b/data/catalog/bachelor_of_arts_in_latin_american_studies/requirements.txt similarity index 100% rename from backend/test_output/bachelor_of_arts_in_latin_american_studies/requirements.txt rename to data/catalog/bachelor_of_arts_in_latin_american_studies/requirements.txt diff --git a/backend/test_output/bachelor_of_arts_in_literature/example.txt b/data/catalog/bachelor_of_arts_in_literature/example.txt similarity index 100% rename from backend/test_output/bachelor_of_arts_in_literature/example.txt rename to data/catalog/bachelor_of_arts_in_literature/example.txt diff --git a/backend/test_output/bachelor_of_arts_in_literature/requirements.txt b/data/catalog/bachelor_of_arts_in_literature/requirements.txt similarity index 100% rename from backend/test_output/bachelor_of_arts_in_literature/requirements.txt rename to data/catalog/bachelor_of_arts_in_literature/requirements.txt diff --git a/backend/test_output/bachelor_of_arts_in_mathematics/example.txt b/data/catalog/bachelor_of_arts_in_mathematics/example.txt similarity index 100% rename from backend/test_output/bachelor_of_arts_in_mathematics/example.txt rename to data/catalog/bachelor_of_arts_in_mathematics/example.txt diff --git a/backend/test_output/bachelor_of_arts_in_mathematics/requirements.txt b/data/catalog/bachelor_of_arts_in_mathematics/requirements.txt similarity index 100% rename from backend/test_output/bachelor_of_arts_in_mathematics/requirements.txt rename to data/catalog/bachelor_of_arts_in_mathematics/requirements.txt diff --git a/backend/test_output/bachelor_of_arts_in_media_arts_and_design/example.txt b/data/catalog/bachelor_of_arts_in_media_arts_and_design/example.txt similarity index 100% rename from backend/test_output/bachelor_of_arts_in_media_arts_and_design/example.txt rename to data/catalog/bachelor_of_arts_in_media_arts_and_design/example.txt diff --git a/backend/test_output/bachelor_of_arts_in_media_arts_and_design/requirements.txt b/data/catalog/bachelor_of_arts_in_media_arts_and_design/requirements.txt similarity index 100% rename from backend/test_output/bachelor_of_arts_in_media_arts_and_design/requirements.txt rename to data/catalog/bachelor_of_arts_in_media_arts_and_design/requirements.txt diff --git a/backend/test_output/bachelor_of_arts_in_philosophy/example.txt b/data/catalog/bachelor_of_arts_in_philosophy/example.txt similarity index 100% rename from backend/test_output/bachelor_of_arts_in_philosophy/example.txt rename to data/catalog/bachelor_of_arts_in_philosophy/example.txt diff --git a/backend/test_output/bachelor_of_arts_in_philosophy/requirements.txt b/data/catalog/bachelor_of_arts_in_philosophy/requirements.txt similarity index 100% rename from backend/test_output/bachelor_of_arts_in_philosophy/requirements.txt rename to data/catalog/bachelor_of_arts_in_philosophy/requirements.txt diff --git a/backend/test_output/bachelor_of_arts_in_physics/example.txt b/data/catalog/bachelor_of_arts_in_physics/example.txt similarity index 100% rename from backend/test_output/bachelor_of_arts_in_physics/example.txt rename to data/catalog/bachelor_of_arts_in_physics/example.txt diff --git a/backend/test_output/bachelor_of_arts_in_physics/requirements.txt b/data/catalog/bachelor_of_arts_in_physics/requirements.txt similarity index 100% rename from backend/test_output/bachelor_of_arts_in_physics/requirements.txt rename to data/catalog/bachelor_of_arts_in_physics/requirements.txt diff --git a/backend/test_output/bachelor_of_arts_in_political_science/example.txt b/data/catalog/bachelor_of_arts_in_political_science/example.txt similarity index 100% rename from backend/test_output/bachelor_of_arts_in_political_science/example.txt rename to data/catalog/bachelor_of_arts_in_political_science/example.txt diff --git a/backend/test_output/bachelor_of_arts_in_political_science/requirements.txt b/data/catalog/bachelor_of_arts_in_political_science/requirements.txt similarity index 100% rename from backend/test_output/bachelor_of_arts_in_political_science/requirements.txt rename to data/catalog/bachelor_of_arts_in_political_science/requirements.txt diff --git a/backend/test_output/bachelor_of_arts_in_public_health/example.txt b/data/catalog/bachelor_of_arts_in_public_health/example.txt similarity index 100% rename from backend/test_output/bachelor_of_arts_in_public_health/example.txt rename to data/catalog/bachelor_of_arts_in_public_health/example.txt diff --git a/backend/test_output/bachelor_of_arts_in_public_health/requirements.txt b/data/catalog/bachelor_of_arts_in_public_health/requirements.txt similarity index 100% rename from backend/test_output/bachelor_of_arts_in_public_health/requirements.txt rename to data/catalog/bachelor_of_arts_in_public_health/requirements.txt diff --git a/backend/test_output/bachelor_of_arts_in_sociology/example.txt b/data/catalog/bachelor_of_arts_in_sociology/example.txt similarity index 100% rename from backend/test_output/bachelor_of_arts_in_sociology/example.txt rename to data/catalog/bachelor_of_arts_in_sociology/example.txt diff --git a/backend/test_output/bachelor_of_arts_in_sociology/requirements.txt b/data/catalog/bachelor_of_arts_in_sociology/requirements.txt similarity index 100% rename from backend/test_output/bachelor_of_arts_in_sociology/requirements.txt rename to data/catalog/bachelor_of_arts_in_sociology/requirements.txt diff --git a/backend/test_output/bachelor_of_arts_in_visual_and_performing_arts/example.txt b/data/catalog/bachelor_of_arts_in_visual_and_performing_arts/example.txt similarity index 100% rename from backend/test_output/bachelor_of_arts_in_visual_and_performing_arts/example.txt rename to data/catalog/bachelor_of_arts_in_visual_and_performing_arts/example.txt diff --git a/backend/test_output/bachelor_of_arts_in_visual_and_performing_arts/requirements.txt b/data/catalog/bachelor_of_arts_in_visual_and_performing_arts/requirements.txt similarity index 100% rename from backend/test_output/bachelor_of_arts_in_visual_and_performing_arts/requirements.txt rename to data/catalog/bachelor_of_arts_in_visual_and_performing_arts/requirements.txt diff --git a/backend/test_output/bachelor_of_science_in_accounting/example.txt b/data/catalog/bachelor_of_science_in_accounting/example.txt similarity index 100% rename from backend/test_output/bachelor_of_science_in_accounting/example.txt rename to data/catalog/bachelor_of_science_in_accounting/example.txt diff --git a/backend/test_output/bachelor_of_science_in_accounting/requirements.txt b/data/catalog/bachelor_of_science_in_accounting/requirements.txt similarity index 100% rename from backend/test_output/bachelor_of_science_in_accounting/requirements.txt rename to data/catalog/bachelor_of_science_in_accounting/requirements.txt diff --git a/backend/test_output/bachelor_of_science_in_actuarial_science/example.txt b/data/catalog/bachelor_of_science_in_actuarial_science/example.txt similarity index 100% rename from backend/test_output/bachelor_of_science_in_actuarial_science/example.txt rename to data/catalog/bachelor_of_science_in_actuarial_science/example.txt diff --git a/backend/test_output/bachelor_of_science_in_actuarial_science/requirements.txt b/data/catalog/bachelor_of_science_in_actuarial_science/requirements.txt similarity index 100% rename from backend/test_output/bachelor_of_science_in_actuarial_science/requirements.txt rename to data/catalog/bachelor_of_science_in_actuarial_science/requirements.txt diff --git a/backend/test_output/bachelor_of_science_in_biochemistry/example.txt b/data/catalog/bachelor_of_science_in_biochemistry/example.txt similarity index 100% rename from backend/test_output/bachelor_of_science_in_biochemistry/example.txt rename to data/catalog/bachelor_of_science_in_biochemistry/example.txt diff --git a/backend/test_output/bachelor_of_science_in_biochemistry/requirements.txt b/data/catalog/bachelor_of_science_in_biochemistry/requirements.txt similarity index 100% rename from backend/test_output/bachelor_of_science_in_biochemistry/requirements.txt rename to data/catalog/bachelor_of_science_in_biochemistry/requirements.txt diff --git a/backend/test_output/bachelor_of_science_in_biomedical_engineering/example.txt b/data/catalog/bachelor_of_science_in_biomedical_engineering/example.txt similarity index 100% rename from backend/test_output/bachelor_of_science_in_biomedical_engineering/example.txt rename to data/catalog/bachelor_of_science_in_biomedical_engineering/example.txt diff --git a/backend/test_output/bachelor_of_science_in_biomedical_engineering/requirements.txt b/data/catalog/bachelor_of_science_in_biomedical_engineering/requirements.txt similarity index 100% rename from backend/test_output/bachelor_of_science_in_biomedical_engineering/requirements.txt rename to data/catalog/bachelor_of_science_in_biomedical_engineering/requirements.txt diff --git a/backend/test_output/bachelor_of_science_in_biomedical_sciences/requirements.txt b/data/catalog/bachelor_of_science_in_biomedical_sciences/requirements.txt similarity index 100% rename from backend/test_output/bachelor_of_science_in_biomedical_sciences/requirements.txt rename to data/catalog/bachelor_of_science_in_biomedical_sciences/requirements.txt diff --git a/backend/test_output/bachelor_of_science_in_business_administration/example.txt b/data/catalog/bachelor_of_science_in_business_administration/example.txt similarity index 100% rename from backend/test_output/bachelor_of_science_in_business_administration/example.txt rename to data/catalog/bachelor_of_science_in_business_administration/example.txt diff --git a/backend/test_output/bachelor_of_science_in_business_administration/requirements.txt b/data/catalog/bachelor_of_science_in_business_administration/requirements.txt similarity index 100% rename from backend/test_output/bachelor_of_science_in_business_administration/requirements.txt rename to data/catalog/bachelor_of_science_in_business_administration/requirements.txt diff --git a/backend/test_output/bachelor_of_science_in_business_analytics_and_artificial_intelligence/example.txt b/data/catalog/bachelor_of_science_in_business_analytics_and_artificial_intelligence/example.txt similarity index 100% rename from backend/test_output/bachelor_of_science_in_business_analytics_and_artificial_intelligence/example.txt rename to data/catalog/bachelor_of_science_in_business_analytics_and_artificial_intelligence/example.txt diff --git a/backend/test_output/bachelor_of_science_in_business_analytics_and_artificial_intelligence/requirements.txt b/data/catalog/bachelor_of_science_in_business_analytics_and_artificial_intelligence/requirements.txt similarity index 100% rename from backend/test_output/bachelor_of_science_in_business_analytics_and_artificial_intelligence/requirements.txt rename to data/catalog/bachelor_of_science_in_business_analytics_and_artificial_intelligence/requirements.txt diff --git a/backend/test_output/bachelor_of_science_in_child_learning_and_development/example.txt b/data/catalog/bachelor_of_science_in_child_learning_and_development/example.txt similarity index 100% rename from backend/test_output/bachelor_of_science_in_child_learning_and_development/example.txt rename to data/catalog/bachelor_of_science_in_child_learning_and_development/example.txt diff --git a/backend/test_output/bachelor_of_science_in_child_learning_and_development/requirements.txt b/data/catalog/bachelor_of_science_in_child_learning_and_development/requirements.txt similarity index 100% rename from backend/test_output/bachelor_of_science_in_child_learning_and_development/requirements.txt rename to data/catalog/bachelor_of_science_in_child_learning_and_development/requirements.txt diff --git a/backend/test_output/bachelor_of_science_in_cognitive_science/example.txt b/data/catalog/bachelor_of_science_in_cognitive_science/example.txt similarity index 100% rename from backend/test_output/bachelor_of_science_in_cognitive_science/example.txt rename to data/catalog/bachelor_of_science_in_cognitive_science/example.txt diff --git a/backend/test_output/bachelor_of_science_in_cognitive_science/requirements.txt b/data/catalog/bachelor_of_science_in_cognitive_science/requirements.txt similarity index 100% rename from backend/test_output/bachelor_of_science_in_cognitive_science/requirements.txt rename to data/catalog/bachelor_of_science_in_cognitive_science/requirements.txt diff --git a/backend/test_output/bachelor_of_science_in_computational_and_geospatial_science/example.txt b/data/catalog/bachelor_of_science_in_computational_and_geospatial_science/example.txt similarity index 100% rename from backend/test_output/bachelor_of_science_in_computational_and_geospatial_science/example.txt rename to data/catalog/bachelor_of_science_in_computational_and_geospatial_science/example.txt diff --git a/backend/test_output/bachelor_of_science_in_computational_and_geospatial_science/requirements.txt b/data/catalog/bachelor_of_science_in_computational_and_geospatial_science/requirements.txt similarity index 100% rename from backend/test_output/bachelor_of_science_in_computational_and_geospatial_science/requirements.txt rename to data/catalog/bachelor_of_science_in_computational_and_geospatial_science/requirements.txt diff --git a/backend/test_output/bachelor_of_science_in_computer_engineering/example.txt b/data/catalog/bachelor_of_science_in_computer_engineering/example.txt similarity index 100% rename from backend/test_output/bachelor_of_science_in_computer_engineering/example.txt rename to data/catalog/bachelor_of_science_in_computer_engineering/example.txt diff --git a/backend/test_output/bachelor_of_science_in_computer_engineering/requirements.txt b/data/catalog/bachelor_of_science_in_computer_engineering/requirements.txt similarity index 100% rename from backend/test_output/bachelor_of_science_in_computer_engineering/requirements.txt rename to data/catalog/bachelor_of_science_in_computer_engineering/requirements.txt diff --git a/backend/test_output/bachelor_of_science_in_computer_information_systems_and_technology/example.txt b/data/catalog/bachelor_of_science_in_computer_information_systems_and_technology/example.txt similarity index 100% rename from backend/test_output/bachelor_of_science_in_computer_information_systems_and_technology/example.txt rename to data/catalog/bachelor_of_science_in_computer_information_systems_and_technology/example.txt diff --git a/backend/test_output/bachelor_of_science_in_computer_information_systems_and_technology/requirements.txt b/data/catalog/bachelor_of_science_in_computer_information_systems_and_technology/requirements.txt similarity index 100% rename from backend/test_output/bachelor_of_science_in_computer_information_systems_and_technology/requirements.txt rename to data/catalog/bachelor_of_science_in_computer_information_systems_and_technology/requirements.txt diff --git a/backend/test_output/bachelor_of_science_in_computer_science/example.txt b/data/catalog/bachelor_of_science_in_computer_science/example.txt similarity index 100% rename from backend/test_output/bachelor_of_science_in_computer_science/example.txt rename to data/catalog/bachelor_of_science_in_computer_science/example.txt diff --git a/backend/test_output/bachelor_of_science_in_computer_science/requirements.txt b/data/catalog/bachelor_of_science_in_computer_science/requirements.txt similarity index 100% rename from backend/test_output/bachelor_of_science_in_computer_science/requirements.txt rename to data/catalog/bachelor_of_science_in_computer_science/requirements.txt diff --git a/backend/test_output/bachelor_of_science_in_cybersecurity_and_risk_management/example.txt b/data/catalog/bachelor_of_science_in_cybersecurity_and_risk_management/example.txt similarity index 100% rename from backend/test_output/bachelor_of_science_in_cybersecurity_and_risk_management/example.txt rename to data/catalog/bachelor_of_science_in_cybersecurity_and_risk_management/example.txt diff --git a/backend/test_output/bachelor_of_science_in_cybersecurity_and_risk_management/requirements.txt b/data/catalog/bachelor_of_science_in_cybersecurity_and_risk_management/requirements.txt similarity index 100% rename from backend/test_output/bachelor_of_science_in_cybersecurity_and_risk_management/requirements.txt rename to data/catalog/bachelor_of_science_in_cybersecurity_and_risk_management/requirements.txt diff --git a/backend/test_output/bachelor_of_science_in_data_science/example.txt b/data/catalog/bachelor_of_science_in_data_science/example.txt similarity index 100% rename from backend/test_output/bachelor_of_science_in_data_science/example.txt rename to data/catalog/bachelor_of_science_in_data_science/example.txt diff --git a/backend/test_output/bachelor_of_science_in_data_science/requirements.txt b/data/catalog/bachelor_of_science_in_data_science/requirements.txt similarity index 100% rename from backend/test_output/bachelor_of_science_in_data_science/requirements.txt rename to data/catalog/bachelor_of_science_in_data_science/requirements.txt diff --git a/backend/test_output/bachelor_of_science_in_electrical_engineering/example.txt b/data/catalog/bachelor_of_science_in_electrical_engineering/example.txt similarity index 100% rename from backend/test_output/bachelor_of_science_in_electrical_engineering/example.txt rename to data/catalog/bachelor_of_science_in_electrical_engineering/example.txt diff --git a/backend/test_output/bachelor_of_science_in_electrical_engineering/requirements.txt b/data/catalog/bachelor_of_science_in_electrical_engineering/requirements.txt similarity index 100% rename from backend/test_output/bachelor_of_science_in_electrical_engineering/requirements.txt rename to data/catalog/bachelor_of_science_in_electrical_engineering/requirements.txt diff --git a/backend/test_output/bachelor_of_science_in_finance/example.txt b/data/catalog/bachelor_of_science_in_finance/example.txt similarity index 100% rename from backend/test_output/bachelor_of_science_in_finance/example.txt rename to data/catalog/bachelor_of_science_in_finance/example.txt diff --git a/backend/test_output/bachelor_of_science_in_finance/requirements.txt b/data/catalog/bachelor_of_science_in_finance/requirements.txt similarity index 100% rename from backend/test_output/bachelor_of_science_in_finance/requirements.txt rename to data/catalog/bachelor_of_science_in_finance/requirements.txt diff --git a/backend/test_output/bachelor_of_science_in_geosciences/example.txt b/data/catalog/bachelor_of_science_in_geosciences/example.txt similarity index 100% rename from backend/test_output/bachelor_of_science_in_geosciences/example.txt rename to data/catalog/bachelor_of_science_in_geosciences/example.txt diff --git a/backend/test_output/bachelor_of_science_in_geosciences/requirements.txt b/data/catalog/bachelor_of_science_in_geosciences/requirements.txt similarity index 100% rename from backend/test_output/bachelor_of_science_in_geosciences/requirements.txt rename to data/catalog/bachelor_of_science_in_geosciences/requirements.txt diff --git a/backend/test_output/bachelor_of_science_in_geospatial_information_sciences/example.txt b/data/catalog/bachelor_of_science_in_geospatial_information_sciences/example.txt similarity index 100% rename from backend/test_output/bachelor_of_science_in_geospatial_information_sciences/example.txt rename to data/catalog/bachelor_of_science_in_geospatial_information_sciences/example.txt diff --git a/backend/test_output/bachelor_of_science_in_geospatial_information_sciences/requirements.txt b/data/catalog/bachelor_of_science_in_geospatial_information_sciences/requirements.txt similarity index 100% rename from backend/test_output/bachelor_of_science_in_geospatial_information_sciences/requirements.txt rename to data/catalog/bachelor_of_science_in_geospatial_information_sciences/requirements.txt diff --git a/backend/test_output/bachelor_of_science_in_global_business/example.txt b/data/catalog/bachelor_of_science_in_global_business/example.txt similarity index 100% rename from backend/test_output/bachelor_of_science_in_global_business/example.txt rename to data/catalog/bachelor_of_science_in_global_business/example.txt diff --git a/backend/test_output/bachelor_of_science_in_global_business/requirements.txt b/data/catalog/bachelor_of_science_in_global_business/requirements.txt similarity index 100% rename from backend/test_output/bachelor_of_science_in_global_business/requirements.txt rename to data/catalog/bachelor_of_science_in_global_business/requirements.txt diff --git a/backend/test_output/bachelor_of_science_in_healthcare_management/example.txt b/data/catalog/bachelor_of_science_in_healthcare_management/example.txt similarity index 100% rename from backend/test_output/bachelor_of_science_in_healthcare_management/example.txt rename to data/catalog/bachelor_of_science_in_healthcare_management/example.txt diff --git a/backend/test_output/bachelor_of_science_in_healthcare_management/requirements.txt b/data/catalog/bachelor_of_science_in_healthcare_management/requirements.txt similarity index 100% rename from backend/test_output/bachelor_of_science_in_healthcare_management/requirements.txt rename to data/catalog/bachelor_of_science_in_healthcare_management/requirements.txt diff --git a/backend/test_output/bachelor_of_science_in_healthcare_studies/example.txt b/data/catalog/bachelor_of_science_in_healthcare_studies/example.txt similarity index 100% rename from backend/test_output/bachelor_of_science_in_healthcare_studies/example.txt rename to data/catalog/bachelor_of_science_in_healthcare_studies/example.txt diff --git a/backend/test_output/bachelor_of_science_in_healthcare_studies/requirements.txt b/data/catalog/bachelor_of_science_in_healthcare_studies/requirements.txt similarity index 100% rename from backend/test_output/bachelor_of_science_in_healthcare_studies/requirements.txt rename to data/catalog/bachelor_of_science_in_healthcare_studies/requirements.txt diff --git a/backend/test_output/bachelor_of_science_in_human_resource_management/example.txt b/data/catalog/bachelor_of_science_in_human_resource_management/example.txt similarity index 100% rename from backend/test_output/bachelor_of_science_in_human_resource_management/example.txt rename to data/catalog/bachelor_of_science_in_human_resource_management/example.txt diff --git a/backend/test_output/bachelor_of_science_in_human_resource_management/requirements.txt b/data/catalog/bachelor_of_science_in_human_resource_management/requirements.txt similarity index 100% rename from backend/test_output/bachelor_of_science_in_human_resource_management/requirements.txt rename to data/catalog/bachelor_of_science_in_human_resource_management/requirements.txt diff --git a/backend/test_output/bachelor_of_science_in_marketing/example.txt b/data/catalog/bachelor_of_science_in_marketing/example.txt similarity index 100% rename from backend/test_output/bachelor_of_science_in_marketing/example.txt rename to data/catalog/bachelor_of_science_in_marketing/example.txt diff --git a/backend/test_output/bachelor_of_science_in_marketing/requirements.txt b/data/catalog/bachelor_of_science_in_marketing/requirements.txt similarity index 100% rename from backend/test_output/bachelor_of_science_in_marketing/requirements.txt rename to data/catalog/bachelor_of_science_in_marketing/requirements.txt diff --git a/backend/test_output/bachelor_of_science_in_mechanical_engineering/example.txt b/data/catalog/bachelor_of_science_in_mechanical_engineering/example.txt similarity index 100% rename from backend/test_output/bachelor_of_science_in_mechanical_engineering/example.txt rename to data/catalog/bachelor_of_science_in_mechanical_engineering/example.txt diff --git a/backend/test_output/bachelor_of_science_in_mechanical_engineering/requirements.txt b/data/catalog/bachelor_of_science_in_mechanical_engineering/requirements.txt similarity index 100% rename from backend/test_output/bachelor_of_science_in_mechanical_engineering/requirements.txt rename to data/catalog/bachelor_of_science_in_mechanical_engineering/requirements.txt diff --git a/backend/test_output/bachelor_of_science_in_molecular_biology/example.txt b/data/catalog/bachelor_of_science_in_molecular_biology/example.txt similarity index 100% rename from backend/test_output/bachelor_of_science_in_molecular_biology/example.txt rename to data/catalog/bachelor_of_science_in_molecular_biology/example.txt diff --git a/backend/test_output/bachelor_of_science_in_molecular_biology/requirements.txt b/data/catalog/bachelor_of_science_in_molecular_biology/requirements.txt similarity index 100% rename from backend/test_output/bachelor_of_science_in_molecular_biology/requirements.txt rename to data/catalog/bachelor_of_science_in_molecular_biology/requirements.txt diff --git a/backend/test_output/bachelor_of_science_in_neuroscience/example.txt b/data/catalog/bachelor_of_science_in_neuroscience/example.txt similarity index 100% rename from backend/test_output/bachelor_of_science_in_neuroscience/example.txt rename to data/catalog/bachelor_of_science_in_neuroscience/example.txt diff --git a/backend/test_output/bachelor_of_science_in_neuroscience/requirements.txt b/data/catalog/bachelor_of_science_in_neuroscience/requirements.txt similarity index 100% rename from backend/test_output/bachelor_of_science_in_neuroscience/requirements.txt rename to data/catalog/bachelor_of_science_in_neuroscience/requirements.txt diff --git a/backend/test_output/bachelor_of_science_in_psychology/example.txt b/data/catalog/bachelor_of_science_in_psychology/example.txt similarity index 100% rename from backend/test_output/bachelor_of_science_in_psychology/example.txt rename to data/catalog/bachelor_of_science_in_psychology/example.txt diff --git a/backend/test_output/bachelor_of_science_in_psychology/requirements.txt b/data/catalog/bachelor_of_science_in_psychology/requirements.txt similarity index 100% rename from backend/test_output/bachelor_of_science_in_psychology/requirements.txt rename to data/catalog/bachelor_of_science_in_psychology/requirements.txt diff --git a/backend/test_output/bachelor_of_science_in_public_affairs/example.txt b/data/catalog/bachelor_of_science_in_public_affairs/example.txt similarity index 100% rename from backend/test_output/bachelor_of_science_in_public_affairs/example.txt rename to data/catalog/bachelor_of_science_in_public_affairs/example.txt diff --git a/backend/test_output/bachelor_of_science_in_public_affairs/requirements.txt b/data/catalog/bachelor_of_science_in_public_affairs/requirements.txt similarity index 100% rename from backend/test_output/bachelor_of_science_in_public_affairs/requirements.txt rename to data/catalog/bachelor_of_science_in_public_affairs/requirements.txt diff --git a/backend/test_output/bachelor_of_science_in_public_policy/example.txt b/data/catalog/bachelor_of_science_in_public_policy/example.txt similarity index 100% rename from backend/test_output/bachelor_of_science_in_public_policy/example.txt rename to data/catalog/bachelor_of_science_in_public_policy/example.txt diff --git a/backend/test_output/bachelor_of_science_in_public_policy/requirements.txt b/data/catalog/bachelor_of_science_in_public_policy/requirements.txt similarity index 100% rename from backend/test_output/bachelor_of_science_in_public_policy/requirements.txt rename to data/catalog/bachelor_of_science_in_public_policy/requirements.txt diff --git a/backend/test_output/bachelor_of_science_in_software_engineering/example.txt b/data/catalog/bachelor_of_science_in_software_engineering/example.txt similarity index 100% rename from backend/test_output/bachelor_of_science_in_software_engineering/example.txt rename to data/catalog/bachelor_of_science_in_software_engineering/example.txt diff --git a/backend/test_output/bachelor_of_science_in_software_engineering/requirements.txt b/data/catalog/bachelor_of_science_in_software_engineering/requirements.txt similarity index 100% rename from backend/test_output/bachelor_of_science_in_software_engineering/requirements.txt rename to data/catalog/bachelor_of_science_in_software_engineering/requirements.txt diff --git a/backend/test_output/bachelor_of_science_in_speech_language_and_hearing_sciences/example.txt b/data/catalog/bachelor_of_science_in_speech_language_and_hearing_sciences/example.txt similarity index 100% rename from backend/test_output/bachelor_of_science_in_speech_language_and_hearing_sciences/example.txt rename to data/catalog/bachelor_of_science_in_speech_language_and_hearing_sciences/example.txt diff --git a/backend/test_output/bachelor_of_science_in_speech_language_and_hearing_sciences/requirements.txt b/data/catalog/bachelor_of_science_in_speech_language_and_hearing_sciences/requirements.txt similarity index 100% rename from backend/test_output/bachelor_of_science_in_speech_language_and_hearing_sciences/requirements.txt rename to data/catalog/bachelor_of_science_in_speech_language_and_hearing_sciences/requirements.txt diff --git a/backend/test_output/bachelor_of_science_in_supply_chain_management_and_analytics/example.txt b/data/catalog/bachelor_of_science_in_supply_chain_management_and_analytics/example.txt similarity index 100% rename from backend/test_output/bachelor_of_science_in_supply_chain_management_and_analytics/example.txt rename to data/catalog/bachelor_of_science_in_supply_chain_management_and_analytics/example.txt diff --git a/backend/test_output/bachelor_of_science_in_supply_chain_management_and_analytics/requirements.txt b/data/catalog/bachelor_of_science_in_supply_chain_management_and_analytics/requirements.txt similarity index 100% rename from backend/test_output/bachelor_of_science_in_supply_chain_management_and_analytics/requirements.txt rename to data/catalog/bachelor_of_science_in_supply_chain_management_and_analytics/requirements.txt diff --git a/backend/test_output/bachelor_of_science_in_systems_engineering/example.txt b/data/catalog/bachelor_of_science_in_systems_engineering/example.txt similarity index 100% rename from backend/test_output/bachelor_of_science_in_systems_engineering/example.txt rename to data/catalog/bachelor_of_science_in_systems_engineering/example.txt diff --git a/backend/test_output/bachelor_of_science_in_systems_engineering/requirements.txt b/data/catalog/bachelor_of_science_in_systems_engineering/requirements.txt similarity index 100% rename from backend/test_output/bachelor_of_science_in_systems_engineering/requirements.txt rename to data/catalog/bachelor_of_science_in_systems_engineering/requirements.txt diff --git a/backend/test_output/bachelor_of_sustainability_planning_and_policy/example.txt b/data/catalog/bachelor_of_sustainability_planning_and_policy/example.txt similarity index 100% rename from backend/test_output/bachelor_of_sustainability_planning_and_policy/example.txt rename to data/catalog/bachelor_of_sustainability_planning_and_policy/example.txt diff --git a/backend/test_output/bachelor_of_sustainability_planning_and_policy/requirements.txt b/data/catalog/bachelor_of_sustainability_planning_and_policy/requirements.txt similarity index 100% rename from backend/test_output/bachelor_of_sustainability_planning_and_policy/requirements.txt rename to data/catalog/bachelor_of_sustainability_planning_and_policy/requirements.txt diff --git a/backend/test_output/bs_in_biology_and_healthcare_management_double_major/requirements.txt b/data/catalog/bs_in_biology_and_healthcare_management_double_major/requirements.txt similarity index 100% rename from backend/test_output/bs_in_biology_and_healthcare_management_double_major/requirements.txt rename to data/catalog/bs_in_biology_and_healthcare_management_double_major/requirements.txt diff --git a/backend/test_output/bs_in_business_analytics_and_artificial_intelligence_and_supply_chain_management_and_analytics_double_major/requirements.txt b/data/catalog/bs_in_business_analytics_and_artificial_intelligence_and_supply_chain_management_and_analytics_double_major/requirements.txt similarity index 100% rename from backend/test_output/bs_in_business_analytics_and_artificial_intelligence_and_supply_chain_management_and_analytics_double_major/requirements.txt rename to data/catalog/bs_in_business_analytics_and_artificial_intelligence_and_supply_chain_management_and_analytics_double_major/requirements.txt diff --git a/backend/test_output/bs_in_economics_and_finance_double_major/requirements.txt b/data/catalog/bs_in_economics_and_finance_double_major/requirements.txt similarity index 100% rename from backend/test_output/bs_in_economics_and_finance_double_major/requirements.txt rename to data/catalog/bs_in_economics_and_finance_double_major/requirements.txt diff --git a/backend/test_output/bs_in_finance_and_economics_double_major/requirements.txt b/data/catalog/bs_in_finance_and_economics_double_major/requirements.txt similarity index 100% rename from backend/test_output/bs_in_finance_and_economics_double_major/requirements.txt rename to data/catalog/bs_in_finance_and_economics_double_major/requirements.txt diff --git a/backend/test_output/bs_in_global_business_and_human_resource_management_double_major/requirements.txt b/data/catalog/bs_in_global_business_and_human_resource_management_double_major/requirements.txt similarity index 100% rename from backend/test_output/bs_in_global_business_and_human_resource_management_double_major/requirements.txt rename to data/catalog/bs_in_global_business_and_human_resource_management_double_major/requirements.txt diff --git a/backend/test_output/bs_in_global_business_and_international_political_economy_double_major/requirements.txt b/data/catalog/bs_in_global_business_and_international_political_economy_double_major/requirements.txt similarity index 100% rename from backend/test_output/bs_in_global_business_and_international_political_economy_double_major/requirements.txt rename to data/catalog/bs_in_global_business_and_international_political_economy_double_major/requirements.txt diff --git a/backend/test_output/bs_in_global_business_and_marketing_double_major/requirements.txt b/data/catalog/bs_in_global_business_and_marketing_double_major/requirements.txt similarity index 100% rename from backend/test_output/bs_in_global_business_and_marketing_double_major/requirements.txt rename to data/catalog/bs_in_global_business_and_marketing_double_major/requirements.txt diff --git a/backend/test_output/bs_in_global_business_and_supply_chain_management_and_analytics_double_major/requirements.txt b/data/catalog/bs_in_global_business_and_supply_chain_management_and_analytics_double_major/requirements.txt similarity index 100% rename from backend/test_output/bs_in_global_business_and_supply_chain_management_and_analytics_double_major/requirements.txt rename to data/catalog/bs_in_global_business_and_supply_chain_management_and_analytics_double_major/requirements.txt diff --git a/backend/test_output/bs_in_healthcare_management_and_biology_double_major/requirements.txt b/data/catalog/bs_in_healthcare_management_and_biology_double_major/requirements.txt similarity index 100% rename from backend/test_output/bs_in_healthcare_management_and_biology_double_major/requirements.txt rename to data/catalog/bs_in_healthcare_management_and_biology_double_major/requirements.txt diff --git a/backend/test_output/bs_in_healthcare_management_and_molecular_biology_double_major/requirements.txt b/data/catalog/bs_in_healthcare_management_and_molecular_biology_double_major/requirements.txt similarity index 100% rename from backend/test_output/bs_in_healthcare_management_and_molecular_biology_double_major/requirements.txt rename to data/catalog/bs_in_healthcare_management_and_molecular_biology_double_major/requirements.txt diff --git a/backend/test_output/bs_in_international_political_economy_and_global_business_double_major/requirements.txt b/data/catalog/bs_in_international_political_economy_and_global_business_double_major/requirements.txt similarity index 100% rename from backend/test_output/bs_in_international_political_economy_and_global_business_double_major/requirements.txt rename to data/catalog/bs_in_international_political_economy_and_global_business_double_major/requirements.txt diff --git a/backend/test_output/bs_in_marketing_and_supply_chain_management_and_analytics_double_major/requirements.txt b/data/catalog/bs_in_marketing_and_supply_chain_management_and_analytics_double_major/requirements.txt similarity index 100% rename from backend/test_output/bs_in_marketing_and_supply_chain_management_and_analytics_double_major/requirements.txt rename to data/catalog/bs_in_marketing_and_supply_chain_management_and_analytics_double_major/requirements.txt diff --git a/backend/test_output/bs_in_molecular_biology_and_healthcare_management_double_major/requirements.txt b/data/catalog/bs_in_molecular_biology_and_healthcare_management_double_major/requirements.txt similarity index 100% rename from backend/test_output/bs_in_molecular_biology_and_healthcare_management_double_major/requirements.txt rename to data/catalog/bs_in_molecular_biology_and_healthcare_management_double_major/requirements.txt diff --git a/backend/test_output/certificate_in_applied_experience_design_and_research/requirements.txt b/data/catalog/certificate_in_applied_experience_design_and_research/requirements.txt similarity index 100% rename from backend/test_output/certificate_in_applied_experience_design_and_research/requirements.txt rename to data/catalog/certificate_in_applied_experience_design_and_research/requirements.txt diff --git a/backend/test_output/certificate_in_behavioral_research_methods/requirements.txt b/data/catalog/certificate_in_behavioral_research_methods/requirements.txt similarity index 100% rename from backend/test_output/certificate_in_behavioral_research_methods/requirements.txt rename to data/catalog/certificate_in_behavioral_research_methods/requirements.txt diff --git a/backend/test_output/certificate_in_biobehavioral_data_science/requirements.txt b/data/catalog/certificate_in_biobehavioral_data_science/requirements.txt similarity index 100% rename from backend/test_output/certificate_in_biobehavioral_data_science/requirements.txt rename to data/catalog/certificate_in_biobehavioral_data_science/requirements.txt diff --git a/backend/test_output/certificate_in_biomedical_data_analytics/example.txt b/data/catalog/certificate_in_biomedical_data_analytics/example.txt similarity index 100% rename from backend/test_output/certificate_in_biomedical_data_analytics/example.txt rename to data/catalog/certificate_in_biomedical_data_analytics/example.txt diff --git a/backend/test_output/certificate_in_biomedical_data_analytics/requirements.txt b/data/catalog/certificate_in_biomedical_data_analytics/requirements.txt similarity index 100% rename from backend/test_output/certificate_in_biomedical_data_analytics/requirements.txt rename to data/catalog/certificate_in_biomedical_data_analytics/requirements.txt diff --git a/backend/test_output/certificate_in_biomedical_sciences/requirements.txt b/data/catalog/certificate_in_biomedical_sciences/requirements.txt similarity index 100% rename from backend/test_output/certificate_in_biomedical_sciences/requirements.txt rename to data/catalog/certificate_in_biomedical_sciences/requirements.txt diff --git a/backend/test_output/certificate_in_compassionate_law_enforcement_active_response_clear/requirements.txt b/data/catalog/certificate_in_compassionate_law_enforcement_active_response_clear/requirements.txt similarity index 100% rename from backend/test_output/certificate_in_compassionate_law_enforcement_active_response_clear/requirements.txt rename to data/catalog/certificate_in_compassionate_law_enforcement_active_response_clear/requirements.txt diff --git a/backend/test_output/certificate_in_creative_writing/requirements.txt b/data/catalog/certificate_in_creative_writing/requirements.txt similarity index 100% rename from backend/test_output/certificate_in_creative_writing/requirements.txt rename to data/catalog/certificate_in_creative_writing/requirements.txt diff --git a/backend/test_output/certificate_in_cyber_defense/requirements.txt b/data/catalog/certificate_in_cyber_defense/requirements.txt similarity index 100% rename from backend/test_output/certificate_in_cyber_defense/requirements.txt rename to data/catalog/certificate_in_cyber_defense/requirements.txt diff --git a/backend/test_output/certificate_in_data_science/example.txt b/data/catalog/certificate_in_data_science/example.txt similarity index 100% rename from backend/test_output/certificate_in_data_science/example.txt rename to data/catalog/certificate_in_data_science/example.txt diff --git a/backend/test_output/certificate_in_data_science/requirements.txt b/data/catalog/certificate_in_data_science/requirements.txt similarity index 100% rename from backend/test_output/certificate_in_data_science/requirements.txt rename to data/catalog/certificate_in_data_science/requirements.txt diff --git a/backend/test_output/certificate_in_geophysics/example.txt b/data/catalog/certificate_in_geophysics/example.txt similarity index 100% rename from backend/test_output/certificate_in_geophysics/example.txt rename to data/catalog/certificate_in_geophysics/example.txt diff --git a/backend/test_output/certificate_in_geophysics/requirements.txt b/data/catalog/certificate_in_geophysics/requirements.txt similarity index 100% rename from backend/test_output/certificate_in_geophysics/requirements.txt rename to data/catalog/certificate_in_geophysics/requirements.txt diff --git a/backend/test_output/certificate_in_holocaust_genocide_and_human_rights/requirements.txt b/data/catalog/certificate_in_holocaust_genocide_and_human_rights/requirements.txt similarity index 100% rename from backend/test_output/certificate_in_holocaust_genocide_and_human_rights/requirements.txt rename to data/catalog/certificate_in_holocaust_genocide_and_human_rights/requirements.txt diff --git a/backend/test_output/certificate_in_innovation_and_entrepreneurship/requirements.txt b/data/catalog/certificate_in_innovation_and_entrepreneurship/requirements.txt similarity index 100% rename from backend/test_output/certificate_in_innovation_and_entrepreneurship/requirements.txt rename to data/catalog/certificate_in_innovation_and_entrepreneurship/requirements.txt diff --git a/backend/test_output/certificate_in_intercultural_communication/requirements.txt b/data/catalog/certificate_in_intercultural_communication/requirements.txt similarity index 100% rename from backend/test_output/certificate_in_intercultural_communication/requirements.txt rename to data/catalog/certificate_in_intercultural_communication/requirements.txt diff --git a/backend/test_output/certificate_in_legal_research_and_writing/requirements.txt b/data/catalog/certificate_in_legal_research_and_writing/requirements.txt similarity index 100% rename from backend/test_output/certificate_in_legal_research_and_writing/requirements.txt rename to data/catalog/certificate_in_legal_research_and_writing/requirements.txt diff --git a/backend/test_output/certificate_in_military_science_leadership/requirements.txt b/data/catalog/certificate_in_military_science_leadership/requirements.txt similarity index 100% rename from backend/test_output/certificate_in_military_science_leadership/requirements.txt rename to data/catalog/certificate_in_military_science_leadership/requirements.txt diff --git a/backend/test_output/certificate_in_nanoscience_and_nanotechnology/requirements.txt b/data/catalog/certificate_in_nanoscience_and_nanotechnology/requirements.txt similarity index 100% rename from backend/test_output/certificate_in_nanoscience_and_nanotechnology/requirements.txt rename to data/catalog/certificate_in_nanoscience_and_nanotechnology/requirements.txt diff --git a/backend/test_output/certificate_in_neural_net_math/requirements.txt b/data/catalog/certificate_in_neural_net_math/requirements.txt similarity index 100% rename from backend/test_output/certificate_in_neural_net_math/requirements.txt rename to data/catalog/certificate_in_neural_net_math/requirements.txt diff --git a/backend/test_output/certificate_in_professional_communication/requirements.txt b/data/catalog/certificate_in_professional_communication/requirements.txt similarity index 100% rename from backend/test_output/certificate_in_professional_communication/requirements.txt rename to data/catalog/certificate_in_professional_communication/requirements.txt diff --git a/backend/test_output/certificate_in_public_service_and_civic_leadership/requirements.txt b/data/catalog/certificate_in_public_service_and_civic_leadership/requirements.txt similarity index 100% rename from backend/test_output/certificate_in_public_service_and_civic_leadership/requirements.txt rename to data/catalog/certificate_in_public_service_and_civic_leadership/requirements.txt diff --git a/backend/test_output/certificate_in_quantum_information/example.txt b/data/catalog/certificate_in_quantum_information/example.txt similarity index 100% rename from backend/test_output/certificate_in_quantum_information/example.txt rename to data/catalog/certificate_in_quantum_information/example.txt diff --git a/backend/test_output/certificate_in_quantum_information/requirements.txt b/data/catalog/certificate_in_quantum_information/requirements.txt similarity index 100% rename from backend/test_output/certificate_in_quantum_information/requirements.txt rename to data/catalog/certificate_in_quantum_information/requirements.txt diff --git a/backend/test_output/certificate_in_semiconductor_technology/requirements.txt b/data/catalog/certificate_in_semiconductor_technology/requirements.txt similarity index 100% rename from backend/test_output/certificate_in_semiconductor_technology/requirements.txt rename to data/catalog/certificate_in_semiconductor_technology/requirements.txt diff --git a/backend/test_output/certificate_in_sustainable_earth_and_environment/example.txt b/data/catalog/certificate_in_sustainable_earth_and_environment/example.txt similarity index 100% rename from backend/test_output/certificate_in_sustainable_earth_and_environment/example.txt rename to data/catalog/certificate_in_sustainable_earth_and_environment/example.txt diff --git a/backend/test_output/certificate_in_sustainable_earth_and_environment/requirements.txt b/data/catalog/certificate_in_sustainable_earth_and_environment/requirements.txt similarity index 100% rename from backend/test_output/certificate_in_sustainable_earth_and_environment/requirements.txt rename to data/catalog/certificate_in_sustainable_earth_and_environment/requirements.txt diff --git a/backend/test_output/certificate_in_sustainable_energy_engineering/requirements.txt b/data/catalog/certificate_in_sustainable_energy_engineering/requirements.txt similarity index 100% rename from backend/test_output/certificate_in_sustainable_energy_engineering/requirements.txt rename to data/catalog/certificate_in_sustainable_energy_engineering/requirements.txt diff --git a/backend/test_output/certificate_in_user_experience_ux_research/requirements.txt b/data/catalog/certificate_in_user_experience_ux_research/requirements.txt similarity index 100% rename from backend/test_output/certificate_in_user_experience_ux_research/requirements.txt rename to data/catalog/certificate_in_user_experience_ux_research/requirements.txt diff --git a/backend/test_output/professional_communication_and_digital_media_studies/example.txt b/data/catalog/professional_communication_and_digital_media_studies/example.txt similarity index 100% rename from backend/test_output/professional_communication_and_digital_media_studies/example.txt rename to data/catalog/professional_communication_and_digital_media_studies/example.txt diff --git a/backend/test_output/professional_communication_and_digital_media_studies/requirements.txt b/data/catalog/professional_communication_and_digital_media_studies/requirements.txt similarity index 100% rename from backend/test_output/professional_communication_and_digital_media_studies/requirements.txt rename to data/catalog/professional_communication_and_digital_media_studies/requirements.txt diff --git a/data/catalog/sources.json b/data/catalog/sources.json new file mode 100644 index 0000000..ee44af8 --- /dev/null +++ b/data/catalog/sources.json @@ -0,0 +1,454 @@ +{ + "ba_in_biology_and_criminology_double_major": { + "url": "https://catalog.utdallas.edu/2025/undergraduate/programs/nsm/biology-criminology", + "title": "BA in Biology and Criminology (Double Major)" + }, + "ba_in_criminology_and_biology_double_major": { + "url": "https://catalog.utdallas.edu/2025/undergraduate/programs/epps/criminology-biology", + "title": "BA in Criminology and Biology (Double Major)" + }, + "ba_in_education_7_12_social_studies_with_a_concentration_in_economics": { + "url": "https://catalog.utdallas.edu/2025/undergraduate/programs/is/education-712-ss-economics", + "title": "BA in Education - 7-12 Social Studies with a concentration in Economics" + }, + "ba_in_education_7_12_social_studies_with_a_concentration_in_geography": { + "url": "https://catalog.utdallas.edu/2025/undergraduate/programs/is/education-712-ss-geography", + "title": "BA in Education - 7-12 Social Studies with a concentration in Geography" + }, + "ba_in_education_7_12_social_studies_with_a_concentration_in_government": { + "url": "https://catalog.utdallas.edu/2025/undergraduate/programs/is/education-712-ss-government", + "title": "BA in Education - 7-12 Social Studies with a concentration in Government" + }, + "ba_in_education_7_12_social_studies_with_a_concentration_in_history": { + "url": "https://catalog.utdallas.edu/2025/undergraduate/programs/is/education-712-ss-history", + "title": "BA in Education - 7-12 Social Studies with a concentration in History" + }, + "ba_in_literature_creative_writing_concentration": { + "url": "https://catalog.utdallas.edu/2025/undergraduate/programs/aht/literature-creative-writing", + "title": "BA in Literature - Creative Writing Concentration" + }, + "ba_in_literature_pre_law_concentration": { + "url": "https://catalog.utdallas.edu/2025/undergraduate/programs/aht/literature-pre-law", + "title": "BA in Literature - Pre-Law Concentration" + }, + "ba_in_literature_professional_and_technical_writing_concentration": { + "url": "https://catalog.utdallas.edu/2025/undergraduate/programs/aht/literature-professional-and-technical-writing", + "title": "BA in Literature - Professional and Technical Writing Concentration" + }, + "ba_in_literature_spanish_concentration": { + "url": "https://catalog.utdallas.edu/2025/undergraduate/programs/aht/literature-spanish", + "title": "BA in Literature - Spanish Concentration" + }, + "ba_in_media_arts_and_design_critical_media_studies_concentration": { + "url": "https://catalog.utdallas.edu/2025/undergraduate/programs/aht/atec-critical-media-studies", + "title": "BA in Media Arts and Design - Critical Media Studies Concentration" + }, + "ba_in_media_arts_and_design_emerging_media_arts_concentration": { + "url": "https://catalog.utdallas.edu/2025/undergraduate/programs/aht/atec-emerging-media-arts", + "title": "BA in Media Arts and Design - Emerging Media Arts Concentration" + }, + "ba_in_visual_and_performing_arts_art_history_concentration": { + "url": "https://catalog.utdallas.edu/2025/undergraduate/programs/aht/visual-and-performing-arts-art-history", + "title": "BA in Visual and Performing Arts - Art History Concentration" + }, + "ba_in_visual_and_performing_arts_communication_studies_concentration": { + "url": "https://catalog.utdallas.edu/2025/undergraduate/programs/aht/visual-and-performing-arts-communication", + "title": "BA in Visual and Performing Arts - Communication Studies Concentration" + }, + "ba_in_visual_and_performing_arts_dance_concentration": { + "url": "https://catalog.utdallas.edu/2025/undergraduate/programs/aht/visual-and-performing-arts-dance", + "title": "BA in Visual and Performing Arts - Dance Concentration" + }, + "ba_in_visual_and_performing_arts_film_concentration": { + "url": "https://catalog.utdallas.edu/2025/undergraduate/programs/aht/visual-and-performing-arts-film", + "title": "BA in Visual and Performing Arts - Film Concentration" + }, + "ba_in_visual_and_performing_arts_music_concentration": { + "url": "https://catalog.utdallas.edu/2025/undergraduate/programs/aht/visual-and-performing-arts-music", + "title": "BA in Visual and Performing Arts - Music Concentration" + }, + "ba_in_visual_and_performing_arts_photo_video_digital_concentration": { + "url": "https://catalog.utdallas.edu/2025/undergraduate/programs/aht/visual-and-performing-arts-photo-video-digital", + "title": "BA in Visual and Performing Arts - Photo Video Digital Concentration" + }, + "ba_in_visual_and_performing_arts_theatre_concentration": { + "url": "https://catalog.utdallas.edu/2025/undergraduate/programs/aht/visual-and-performing-arts-theatre", + "title": "BA in Visual and Performing Arts - Theatre Concentration" + }, + "ba_in_visual_and_performing_arts_visual_arts_concentration": { + "url": "https://catalog.utdallas.edu/2025/undergraduate/programs/aht/visual-and-performing-arts-visual-arts", + "title": "BA in Visual and Performing Arts - Visual Arts Concentration" + }, + "bachelor_of_arts_in_american_studies": { + "url": "https://catalog.utdallas.edu/2025/undergraduate/programs/is/american-studies", + "title": "Bachelor of Arts in American Studies" + }, + "bachelor_of_arts_in_animation_and_games": { + "url": "https://catalog.utdallas.edu/2025/undergraduate/programs/aht/animation-and-games", + "title": "Bachelor of Arts in Animation and Games" + }, + "bachelor_of_arts_in_biology": { + "url": "https://catalog.utdallas.edu/2025/undergraduate/programs/nsm/biology", + "title": "Bachelor of Arts in Biology" + }, + "bachelor_of_arts_in_chemistry": { + "url": "https://catalog.utdallas.edu/2025/undergraduate/programs/nsm/chemistry", + "title": "Bachelor of Arts in Chemistry" + }, + "bachelor_of_arts_in_criminology": { + "url": "https://catalog.utdallas.edu/2025/undergraduate/programs/epps/criminology", + "title": "Bachelor of Arts in Criminology" + }, + "bachelor_of_arts_in_economics": { + "url": "https://catalog.utdallas.edu/2025/undergraduate/programs/epps/economics", + "title": "Bachelor of Arts in Economics" + }, + "bachelor_of_arts_in_education": { + "url": "https://catalog.utdallas.edu/2025/undergraduate/programs/is/education", + "title": "Bachelor of Arts in Education" + }, + "bachelor_of_arts_in_history": { + "url": "https://catalog.utdallas.edu/2025/undergraduate/programs/aht/history", + "title": "Bachelor of Arts in History" + }, + "bachelor_of_arts_in_interdisciplinary_studies": { + "url": "https://catalog.utdallas.edu/2025/undergraduate/programs/is/interdisciplinary-studies", + "title": "Bachelor of Arts in Interdisciplinary Studies" + }, + "bachelor_of_arts_in_international_political_economy": { + "url": "https://catalog.utdallas.edu/2025/undergraduate/programs/epps/international-political-economy", + "title": "Bachelor of Arts in International Political Economy" + }, + "bachelor_of_arts_in_latin_american_studies": { + "url": "https://catalog.utdallas.edu/2025/undergraduate/programs/aht/latin-american-studies", + "title": "Bachelor of Arts in Latin American Studies" + }, + "bachelor_of_arts_in_literature": { + "url": "https://catalog.utdallas.edu/2025/undergraduate/programs/aht/literature", + "title": "Bachelor of Arts in Literature" + }, + "bachelor_of_arts_in_mathematics": { + "url": "https://catalog.utdallas.edu/2025/undergraduate/programs/nsm/mathematics", + "title": "Bachelor of Arts in Mathematics" + }, + "bachelor_of_arts_in_media_arts_and_design": { + "url": "https://catalog.utdallas.edu/2025/undergraduate/programs/aht/atec", + "title": "Bachelor of Arts in Media Arts and Design" + }, + "bachelor_of_arts_in_philosophy": { + "url": "https://catalog.utdallas.edu/2025/undergraduate/programs/aht/philosophy", + "title": "Bachelor of Arts in Philosophy" + }, + "bachelor_of_arts_in_physics": { + "url": "https://catalog.utdallas.edu/2025/undergraduate/programs/nsm/physics", + "title": "Bachelor of Arts in Physics" + }, + "bachelor_of_arts_in_political_science": { + "url": "https://catalog.utdallas.edu/2025/undergraduate/programs/epps/political-science", + "title": "Bachelor of Arts in Political Science" + }, + "bachelor_of_arts_in_public_health": { + "url": "https://catalog.utdallas.edu/2025-undergraduate/programs/epps/public-health", + "title": "Bachelor of Arts in Public Health" + }, + "bachelor_of_arts_in_sociology": { + "url": "https://catalog.utdallas.edu/2025/undergraduate/programs/epps/sociology", + "title": "Bachelor of Arts in Sociology" + }, + "bachelor_of_arts_in_visual_and_performing_arts": { + "url": "https://catalog.utdallas.edu/2025/undergraduate/programs/aht/visual-and-performing-arts-interdisciplinary-arts", + "title": "Bachelor of Arts in Visual and Performing Arts" + }, + "bachelor_of_science_in_accounting": { + "url": "https://catalog.utdallas.edu/2025/undergraduate/programs/jsom/accounting", + "title": "Bachelor of Science in Accounting" + }, + "bachelor_of_science_in_actuarial_science": { + "url": "https://catalog.utdallas.edu/2025/undergraduate/programs/nsm/actuarial-science", + "title": "Bachelor of Science in Actuarial Science" + }, + "bachelor_of_science_in_biochemistry": { + "url": "https://catalog.utdallas.edu/2025/undergraduate/programs/nsm/biochemistry", + "title": "Bachelor of Science in Biochemistry" + }, + "bachelor_of_science_in_biomedical_engineering": { + "url": "https://catalog.utdallas.edu/2025/undergraduate/programs/ecs/biomedical-engineering", + "title": "Bachelor of Science in Biomedical Engineering" + }, + "bachelor_of_science_in_biomedical_sciences": { + "url": "https://catalog.utdallas.edu/2025/undergraduate/programs/nsm/biomedical-sciences", + "title": "Bachelor of Science in Biomedical Sciences" + }, + "bachelor_of_science_in_business_administration": { + "url": "https://catalog.utdallas.edu/2025/undergraduate/programs/jsom/business-administration", + "title": "Bachelor of Science in Business Administration" + }, + "bachelor_of_science_in_business_analytics_and_artificial_intelligence": { + "url": "https://catalog.utdallas.edu/2025/undergraduate/programs/jsom/business-analytics", + "title": "Bachelor of Science in Business Analytics and Artificial Intelligence" + }, + "bachelor_of_science_in_child_learning_and_development": { + "url": "https://catalog.utdallas.edu/2025/undergraduate/programs/bbs/child-learning-and-development", + "title": "Bachelor of Science in Child Learning and Development" + }, + "bachelor_of_science_in_cognitive_science": { + "url": "https://catalog.utdallas.edu/2025/undergraduate/programs/bbs/cognitive-science", + "title": "Bachelor of Science in Cognitive Science" + }, + "bachelor_of_science_in_computational_and_geospatial_science": { + "url": "https://catalog.utdallas.edu/2025-undergraduate/programs/ecs/computational-and-geospatial-sciences", + "title": "Bachelor of Science in Computational and Geospatial Science" + }, + "bachelor_of_science_in_computer_engineering": { + "url": "https://catalog.utdallas.edu/2025/undergraduate/programs/ecs/computer-engineering", + "title": "Bachelor of Science in Computer Engineering" + }, + "bachelor_of_science_in_computer_information_systems_and_technology": { + "url": "https://catalog.utdallas.edu/2025/undergraduate/programs/jsom/computer-information-systems-and-technology", + "title": "Bachelor of Science in Computer Information Systems and Technology" + }, + "bachelor_of_science_in_computer_science": { + "url": "https://catalog.utdallas.edu/2025/undergraduate/programs/ecs/computer-science", + "title": "Bachelor of Science in Computer Science" + }, + "bachelor_of_science_in_cybersecurity_and_risk_management": { + "url": "https://catalog.utdallas.edu/2025/undergraduate/programs/jsom/cybersecurity-and-risk-management", + "title": "Bachelor of Science in Cybersecurity and Risk Management" + }, + "bachelor_of_science_in_data_science": { + "url": "https://catalog.utdallas.edu/2025/undergraduate/programs/ecs/data-science", + "title": "Bachelor of Science in Data Science" + }, + "bachelor_of_science_in_electrical_engineering": { + "url": "https://catalog.utdallas.edu/2025/undergraduate/programs/ecs/electrical-engineering", + "title": "Bachelor of Science in Electrical Engineering" + }, + "bachelor_of_science_in_finance": { + "url": "https://catalog.utdallas.edu/2025/undergraduate/programs/jsom/finance", + "title": "Bachelor of Science in Finance" + }, + "bachelor_of_science_in_geosciences": { + "url": "https://catalog.utdallas.edu/2025/undergraduate/programs/nsm/geosciences", + "title": "Bachelor of Science in Geosciences" + }, + "bachelor_of_science_in_geospatial_information_sciences": { + "url": "https://catalog.utdallas.edu/2025/undergraduate/programs/epps/geospatial-information-science", + "title": "Bachelor of Science in Geospatial Information Sciences" + }, + "bachelor_of_science_in_global_business": { + "url": "https://catalog.utdallas.edu/2025/undergraduate/programs/jsom/global-business", + "title": "Bachelor of Science in Global Business" + }, + "bachelor_of_science_in_healthcare_management": { + "url": "https://catalog.utdallas.edu/2025/undergraduate/programs/jsom/healthcare-management", + "title": "Bachelor of Science in Healthcare Management" + }, + "bachelor_of_science_in_healthcare_studies": { + "url": "https://catalog.utdallas.edu/2025/undergraduate/programs/is/healthcare", + "title": "Bachelor of Science in Healthcare Studies" + }, + "bachelor_of_science_in_human_resource_management": { + "url": "https://catalog.utdallas.edu/2025-undergraduate/programs/jsom/human-resource-management", + "title": "Bachelor of Science in Human Resource Management" + }, + "bachelor_of_science_in_marketing": { + "url": "https://catalog.utdallas.edu/2025/undergraduate/programs/jsom/marketing", + "title": "Bachelor of Science in Marketing" + }, + "bachelor_of_science_in_mechanical_engineering": { + "url": "https://catalog.utdallas.edu/2025/undergraduate/programs/ecs/mechanical-engineering", + "title": "Bachelor of Science in Mechanical Engineering" + }, + "bachelor_of_science_in_molecular_biology": { + "url": "https://catalog.utdallas.edu/2025/undergraduate/programs/nsm/molecular-biology", + "title": "Bachelor of Science in Molecular Biology" + }, + "bachelor_of_science_in_neuroscience": { + "url": "https://catalog.utdallas.edu/2025/undergraduate/programs/bbs/neuroscience", + "title": "Bachelor of Science in Neuroscience" + }, + "bachelor_of_science_in_psychology": { + "url": "https://catalog.utdallas.edu/2025/undergraduate/programs/bbs/psychology", + "title": "Bachelor of Science in Psychology" + }, + "bachelor_of_science_in_public_affairs": { + "url": "https://catalog.utdallas.edu/2025/undergraduate/programs/epps/public-affairs", + "title": "Bachelor of Science in Public Affairs" + }, + "bachelor_of_science_in_public_policy": { + "url": "https://catalog.utdallas.edu/2025/undergraduate/programs/epps/public-policy", + "title": "Bachelor of Science in Public Policy" + }, + "bachelor_of_science_in_software_engineering": { + "url": "https://catalog.utdallas.edu/2025/undergraduate/programs/ecs/software-engineering", + "title": "Bachelor of Science in Software Engineering" + }, + "bachelor_of_science_in_speech_language_and_hearing_sciences": { + "url": "https://catalog.utdallas.edu/2025/undergraduate/programs/bbs/speech-language-and-hearing-sciences", + "title": "Bachelor of Science in Speech, Language, and Hearing Sciences" + }, + "bachelor_of_science_in_supply_chain_management_and_analytics": { + "url": "https://catalog.utdallas.edu/2025/undergraduate/programs/jsom/supply-chain-management", + "title": "Bachelor of Science in Supply Chain Management and Analytics" + }, + "bachelor_of_science_in_systems_engineering": { + "url": "https://catalog.utdallas.edu/2025/undergraduate/programs/ecs/systems-engineering", + "title": "Bachelor of Science in Systems Engineering" + }, + "bachelor_of_sustainability_planning_and_policy": { + "url": "https://catalog.utdallas.edu/2025/undergraduate/programs/epps/sustainability-planning-and-policy", + "title": "Bachelor of Sustainability Planning and Policy" + }, + "bs_in_biology_and_healthcare_management_double_major": { + "url": "https://catalog.utdallas.edu/2025/undergraduate/programs/nsm/biology-healthcare-management", + "title": "BS in Biology and Healthcare Management (Double Major)" + }, + "bs_in_business_analytics_and_artificial_intelligence_and_supply_chain_management_and_analytics_double_major": { + "url": "https://catalog.utdallas.edu/2025-undergraduate/programs/jsom/business-analytics-supply-chain-management", + "title": "BS in Business Analytics and Artificial Intelligence and Supply Chain Management and Analytics (Double Major)" + }, + "bs_in_economics_and_finance_double_major": { + "url": "https://catalog.utdallas.edu/2025/undergraduate/programs/epps/economics-finance", + "title": "BS in Economics and Finance (Double Major)" + }, + "bs_in_finance_and_economics_double_major": { + "url": "https://catalog.utdallas.edu/2025/undergraduate/programs/jsom/finance-economics", + "title": "BS in Finance and Economics (Double Major)" + }, + "bs_in_global_business_and_human_resource_management_double_major": { + "url": "https://catalog.utdallas.edu/2025/undergraduate/programs/jsom/global-business-human-resource-management", + "title": "BS in Global Business and Human Resource Management (Double Major)" + }, + "bs_in_global_business_and_international_political_economy_double_major": { + "url": "https://catalog.utdallas.edu/2025/undergraduate/programs/jsom/global-business-international-political-economy", + "title": "BS in Global Business and International Political Economy (Double Major)" + }, + "bs_in_global_business_and_marketing_double_major": { + "url": "https://catalog.utdallas.edu/2025/undergraduate/programs/jsom/global-business-marketing", + "title": "BS in Global Business and Marketing (Double Major)" + }, + "bs_in_global_business_and_supply_chain_management_and_analytics_double_major": { + "url": "https://catalog.utdallas.edu/2025/undergraduate/programs/jsom/global-business-supply-chain-management", + "title": "BS in Global Business and Supply Chain Management and Analytics (Double Major)" + }, + "bs_in_healthcare_management_and_biology_double_major": { + "url": "https://catalog.utdallas.edu/2025/undergraduate/programs/jsom/healthcare-management-biology", + "title": "BS in Healthcare Management and Biology (Double Major)" + }, + "bs_in_healthcare_management_and_molecular_biology_double_major": { + "url": "https://catalog.utdallas.edu/2025/undergraduate/programs/jsom/healthcare-management-molecular-biology", + "title": "BS in Healthcare Management and Molecular Biology (Double Major)" + }, + "bs_in_international_political_economy_and_global_business_double_major": { + "url": "https://catalog.utdallas.edu/2025/undergraduate/programs/epps/international-political-economy-global-business", + "title": "BS in International Political Economy and Global Business (Double Major)" + }, + "bs_in_marketing_and_supply_chain_management_and_analytics_double_major": { + "url": "https://catalog.utdallas.edu/2025-undergraduate/programs/jsom/marketing-supply-chain-management", + "title": "BS in Marketing and Supply Chain Management and Analytics (Double Major)" + }, + "bs_in_molecular_biology_and_healthcare_management_double_major": { + "url": "https://catalog.utdallas.edu/2025/undergraduate/programs/nsm/molecular-biology-healthcare-management", + "title": "BS in Molecular Biology and Healthcare Management (Double Major)" + }, + "certificate_in_applied_experience_design_and_research": { + "url": "https://catalog.utdallas.edu/2025/undergraduate/programs/aht/certificates#cert-applied-experience-design-and-research", + "title": "Certificate in Applied Experience Design and Research" + }, + "certificate_in_behavioral_research_methods": { + "url": "https://catalog.utdallas.edu/2025/undergraduate/programs/bbs/certificates#cert-in-user-behavioral-research-methods", + "title": "Certificate in Behavioral Research Methods" + }, + "certificate_in_biobehavioral_data_science": { + "url": "https://catalog.utdallas.edu/2025/undergraduate/programs/bbs/certificates#cert-in-user-biobehavioral-data-science", + "title": "Certificate in Biobehavioral Data Science" + }, + "certificate_in_biomedical_data_analytics": { + "url": "https://catalog.utdallas.edu/2025/undergraduate/programs/nsm/mathematics#certificate-in-biomedical-data-analytics", + "title": "Certificate in Biomedical Data Analytics" + }, + "certificate_in_biomedical_sciences": { + "url": "https://catalog.utdallas.edu/2025/undergraduate/programs/nsm/biomedical-sciences-certificate", + "title": "Certificate in Biomedical Sciences" + }, + "certificate_in_compassionate_law_enforcement_active_response_clear": { + "url": "https://catalog.utdallas.edu/2025/undergraduate/programs/epps/certificates", + "title": "Certificate in Compassionate Law Enforcement Active Response (CLEAR)" + }, + "certificate_in_creative_writing": { + "url": "https://catalog.utdallas.edu/2025-undergraduate/programs/aht/certificates#cert-in-creative-writing", + "title": "Certificate in Creative Writing" + }, + "certificate_in_cyber_defense": { + "url": "https://catalog.utdallas.edu/2025/undergraduate/programs/ecs/certificates#certificate-cyber-defense", + "title": "Certificate in Cyber Defense" + }, + "certificate_in_data_science": { + "url": "https://catalog.utdallas.edu/2025/undergraduate/programs/nsm/data-science#certificate-in-data-science", + "title": "Certificate in Data Science" + }, + "certificate_in_geophysics": { + "url": "https://catalog.utdallas.edu/2025/undergraduate/programs/nsm/geosciences#certificate-in-geophysics", + "title": "Certificate in Geophysics" + }, + "certificate_in_holocaust_genocide_and_human_rights": { + "url": "https://catalog.utdallas.edu/2025-undergraduate/programs/aht/certificates#cert-in-holocaust-genocide-and-human-rights", + "title": "Certificate in Holocaust, Genocide, and Human Rights" + }, + "certificate_in_innovation_and_entrepreneurship": { + "url": "https://catalog.utdallas.edu/2025/undergraduate/programs/jsom/certificates", + "title": "Certificate in Innovation and Entrepreneurship" + }, + "certificate_in_intercultural_communication": { + "url": "https://catalog.utdallas.edu/2025-undergraduate/programs/aht/certificates#cert-in-intercultural-communication", + "title": "Certificate in Intercultural Communication" + }, + "certificate_in_legal_research_and_writing": { + "url": "https://catalog.utdallas.edu/2025-undergraduate/programs/aht/certificates#cert-in-legal-research-and-writing", + "title": "Certificate in Legal Research and Writing" + }, + "certificate_in_military_science_leadership": { + "url": "https://catalog.utdallas.edu/2025/undergraduate/programs/epps/certificates#certificate-in-military-science-leadership", + "title": "Certificate in Military Science Leadership" + }, + "certificate_in_nanoscience_and_nanotechnology": { + "url": "https://catalog.utdallas.edu/2025/undergraduate/programs/ecs/certificates#certificate-nanoscience-and-nanotech", + "title": "Certificate in Nanoscience and Nanotechnology" + }, + "certificate_in_neural_net_math": { + "url": "https://catalog.utdallas.edu/2025/undergraduate/programs/bbs/certificates#cert-in-user-neural-net-math", + "title": "Certificate in Neural Net Math" + }, + "certificate_in_professional_communication": { + "url": "https://catalog.utdallas.edu/2025/undergraduate/programs/aht/certificates#cert-in-professional-communication", + "title": "Certificate in Professional Communication" + }, + "certificate_in_public_service_and_civic_leadership": { + "url": "https://catalog.utdallas.edu/2025/undergraduate/programs/epps/certificates#certificate-in-public-service-civic-leadership", + "title": "Certificate in Public Service and Civic Leadership" + }, + "certificate_in_quantum_information": { + "url": "https://catalog.utdallas.edu/2025/undergraduate/programs/nsm/physics#certificate-in-quantum-information", + "title": "Certificate in Quantum Information" + }, + "certificate_in_semiconductor_technology": { + "url": "https://catalog.utdallas.edu/2025/undergraduate/programs/ecs/certificates#certificate-semiconductor-technology", + "title": "Certificate in Semiconductor Technology" + }, + "certificate_in_sustainable_earth_and_environment": { + "url": "https://catalog.utdallas.edu/2025/undergraduate/programs/nsm/geosciences#certificate-in-sustainable-earth-and-environment", + "title": "Certificate in Sustainable Earth and Environment" + }, + "certificate_in_sustainable_energy_engineering": { + "url": "https://catalog.utdallas.edu/2025/undergraduate/programs/ecs/certificates#certificate-sustainable-energy", + "title": "Certificate in Sustainable Energy Engineering" + }, + "certificate_in_user_experience_ux_research": { + "url": "https://catalog.utdallas.edu/2025/undergraduate/programs/bbs/certificates#cert-in-user-experience-ux-research", + "title": "Certificate in User Experience (UX) Research" + }, + "professional_communication_and_digital_media_studies": { + "url": "https://catalog.utdallas.edu/2025/undergraduate/programs/aht/professional-communication-and-digital-media-studies", + "title": "Professional Communication and Digital Media Studies" + } +} diff --git a/data/housing/1877_university_housing_updates.txt b/data/housing/1877_university_housing_updates.txt new file mode 100644 index 0000000..6d764d3 --- /dev/null +++ b/data/housing/1877_university_housing_updates.txt @@ -0,0 +1,37 @@ +URL: https://housing.utdallas.edu/1877/university-housing-updates +Title: University Housing Updates - University Housing | The University of Texas at Dallas +Depth: 1 + +------------------------------------------------------------ + +10/7/2025 Please make note of the following housing application deadlines. Applications will be available in the MyHousing portal. +Oct. 1, 2025: Spring 2026 Application opens. (New residents only. For Spring 2026 housing only.) +Nov. 3, 2025:  First-Year Housing Application opens. (First-year students only. For Fall 2026 – Spring 2027.)  +Jan 20, 2026: Sign up to be notified when the New Resident Housing Application opens for Fall 2026 – Spring 2027. (New residents only. For Fall 2026 – Spring 2027.)  +Jan. 20 – Mar. 6, 2026:  Renewal and Sign-Up. (Returning residents only. For Fall 2026 – Spring 2027.) +Jan. 20-23, 2026: Renewal Application available. +Feb. 2-6, 2026: Intent to Sign Up Application available. +Feb. 16-20, 2026: Roommate Groups Application available. +Mar. 2-6, 2026: Room Selection Application available.  +Apr. 15, 2026: First-Year Housing Application priority deadline. (First-year students only. For the Fall 2026 – Spring 2027 academic year.)  +6/26/2025 The New Resident Application opens on June 30, 2025 at 10 a.m. Log in to the MyHousing Portal to apply. +7/31/2025 View University Housing’s updated Electric Recreational Mobility Device Policy. +3/24/2025 Due to the number of students on our waitlist, the opening of the New Resident Application has been delayed. Sign up to be notified when it opens. +3/10/2025 Apr. 15, 2025 is the First-Year Housing Application priority deadline. (First-year students only. For the 2025-2026 academic year.) +3/10/2025 Jan. 12- Mar. 7, 2025: Renewal and Sign-Up. (Returning residents only. For the 2025-2026 academic year.) +Jan. 13-17, 2025: Renewal Application available. +Feb. 3-7, 2025: Intent to Sign Up Application available. +Feb. 17-21, 2025: Roommate Groups Application available. +Mar. 3-7, 2025: Room Selection Application available. +12/18/2024 Starting January 1, 2025, the cancellation policy for the remainder of the 2024-2025 housing agreement will be moved to a flat fee cancellation policy. Click here for more information. +12/14/2024 View University Housing’s winter break hours here. +9/19/2024 Please make note of the following housing application deadlines. Applications will be available in the MyHousing portal. +Oct. 1, 2024: Spring 2025 Application opens. (New residents only. For Spring 2025 housing only.) +Nov. 1, 2024: First-Year Housing Application opens. (First-year students only. For the 2025-2026 academic year.) +Jan. 12- Mar. 7, 2025: Renewal and Sign-Up. (Returning residents only. For the 2025-2026 academic year.) +Jan. 21-24, 2025: Renewal Application available. +Feb. 3-7, 2025: Intent to Sign Up Application available. +Feb. 17-21, 2025: Roommate Groups Application available. +Mar. 3-7, 2025: Room Selection Application available. +Mar. 24, 2025: TENTATIVE: New Resident Housing Application opens. +Apr. 15, 2025: First-Year Housing Application priority deadline. (First-year students only. For the 2025-2026 academic year.) \ No newline at end of file diff --git a/data/housing/canyon_creek_heights_pricing_and_floorplans.txt b/data/housing/canyon_creek_heights_pricing_and_floorplans.txt new file mode 100644 index 0000000..4fdb213 --- /dev/null +++ b/data/housing/canyon_creek_heights_pricing_and_floorplans.txt @@ -0,0 +1,62 @@ +URL: https://housing.utdallas.edu/canyon-creek-heights-pricing-and-floorplans +Title: Canyon Creek Heights Pricing and Floor Plans | University Housing | UTD +Depth: 2 + +------------------------------------------------------------ + +Canyon Creek Heights Pricing and Floor Plans +Pricing +Floor Plan1 Building Bed/Bath Square Feet1 Price Per Bed, Per Semester2 Price Per Bed +SUMMER 2026 ONLY3 +1×1 Canyon Creek Heights South 1/1 416 $5,450 $3,586 +2×2 Canyon Creek Heights South 2/2 602 $4,625 $3,043 +2×2 Canyon Creek Heights North 2/2 739 $4,575 $3,010 +4×2 Canyon Creek Heights North 4/2 886 $4,075 $2,681 +Canyon Creek Heights Pricing and Floor Plans +1 Size information of floor plan is average for typical apartment. Exact dimensions, square footage and layouts vary. +2 The following rates are planned for on-campus University Housing options for the 2025-2026 academic year. Rates are subject to change. Price includes all utilities. +3 The following rates are planned for on-campus University Housing options for summer 2026. Summer rates are only applicable to UT Dallas students who will have a fall 2026 and spring 2027 agreement; therefore, no summer only housing is permitted. Rates are subject to change. Price includes all utilities. Rates are not applicable to RCCS summer corporate intern guests. +Floor Plans + +Apartment is furnished. Measurements are approximate. Floor plans may vary. + +Canyon Creek Heights South – 1×1 + +Private bedroom – $5,450 per semester, all utilities included; academic year contract, summer additional. + +Canyon Creek Heights South – 2×2 + +Private bedroom – $4,625 per semester, all utilities included; academic year contract, summer additional. + +Canyon Creek Heights North – 2×2 + +Private bedroom – $4,575 per semester, all utilities included; academic year contract, summer additional. + +Non-First Year Students: While this floorplan includes two beds, only one student shall occupy each bedroom. +First-Year Overflow Housing: Temporary overflow housing places two students per bedroom (four students per apartment). Private bedrooms are not available. +Canyon Creek Heights North – 4×2 + +Private bedroom – $4,075 per semester, all utilities included; academic year contract, summer additional. + +RELATED LINKS + +Features & Amenities + +Resident Handbook + +Cancellation Policy (pdf) + +Housing Agreements + +All housing agreements are for the complete academic year. + +Students who live in Canyon Creek Heights will sign an academic year lease (mid-August through mid-May), with an option to participate in summer housing if desired. + +All housing charges associated with assignments in these buildings will be processed through the Bursar Office. Housing fees for the full housing agreement period will be divided into two equal charges. Your University account will be charged once prior to the fall semester and once prior to the spring semester, in accordance with Bursar timelines. + +Students who move on campus prior to census day will be charged the full amount of housing for the semester. Charges will not be pro-rated until these dates: + +Spring 2026: Feb. 4 +Summer 2026: June 16 +Fall 2026: Sept. 9 +Spring 2027: Feb. 3 \ No newline at end of file diff --git a/data/housing/category_uncategorized.txt b/data/housing/category_uncategorized.txt new file mode 100644 index 0000000..04c9452 --- /dev/null +++ b/data/housing/category_uncategorized.txt @@ -0,0 +1,21 @@ +URL: https://housing.utdallas.edu/category/uncategorized +Title: Uncategorized - University Housing | The University of Texas at Dallas +Depth: 2 + +------------------------------------------------------------ + +Move-In Maps +July 12, 2024 | Updated: June 17, 2025 +University Housing Updates +June 13, 2023 | Updated: October 07, 2025 +Dining Hours – Feb. 7 – May 13, 2022 +January 14, 2022 +Dining Hours Jan. 18 – Feb. 6, 2022 +January 14, 2022 +Winter Break Hours – Fall 2021 +December 15, 2021 +November 17, 2021 +Vaccine Self-Report Form +August 2, 2021 +URGENT: COVID-19 Protocols for Students Traveling to the U.S. from International Destinations +July 23, 2021 \ No newline at end of file diff --git a/data/housing/contact.txt b/data/housing/contact.txt new file mode 100644 index 0000000..c13b5f9 --- /dev/null +++ b/data/housing/contact.txt @@ -0,0 +1,97 @@ +URL: https://housing.utdallas.edu/contact +Title: Housing Offices | UTD +Depth: 1 + +------------------------------------------------------------ + +Housing Offices +Housing Offices +Housing Leadership +Staff Listings +University Housing Office + +Andromeda Hall +955 N Loop Rd. +Richardson, TX 75080 +972-883-7430 +Monday-Thursday: 8 a.m.-6 p.m. +Friday: 8 a.m.-5 p.m. + +University Village Housing Office + +2800 Waterview Pkwy Ste. 200 +Richardson, TX 75080 +972-883-7300 +Monday-Friday: 10 a.m.- 7 p.m. +Saturday: 1 p.m.- 5 p.m. + +Maintenance + +University Housing residents may place a request for service online via MyHousing. + +Residents may also phone or walk in to place their request. Emergency requests must be reported in person or by calling the appropriate building listed. + +Housing Tours + +The Office of Admission and Enrollment offers walking tours featuring on-campus housing. + +Residential Life + +Residential Life serves the students who live in University Housing. For questions concerning applications and assignments, contact the University Housing Office. View Residential Life contacts here. + +Capella Hall + +2851 Rutford Ave. +Richardson, TX 75080 +972-883-5241 +Monday-Sunday: Open 24 hours + +Vega Hall + +950 North Loop Rd. +Richardson, TX 75080 +972-883-7341 +Monday-Sunday: 8 a.m.- 2 a.m. + +Sirius Hall + +2801 Rutford Ave. +Richardson, TX 75080 +972-883-5561 +Hours: Monday-Sunday: 8 a.m.- 2 a.m. + +Helix Hall + +931 Drive E +Richardson, TX 75080 +972-883-7361 +Hours: Monday-Sunday: 8 a.m.- 2 a.m. + +Andromeda Hall + +955 North Loop Rd. +Richardson, TX 75080 +972-883-7410 +Hours: Monday-Sunday: Open 24 hours + +Canyon Creek Heights North + +990 Loop Rd. SW +Richardson, TX 75080 +972-883-7391 +Monday-Sunday: 9 a.m.- 9 p.m. + +Canyon Creek Heights South + +989 Loop Rd. SW +Richardson, TX 75080 +972-883-7381 +Monday-Sunday: 9 a.m.- 9 p.m. + +University Village Housing Office + +2800 Waterview Pkwy Ste. 200 +Richardson, TX 75080 +972-883-7300 +Monday-Friday: 10 a.m.- 7 p.m. +Saturday: 1 p.m.- 5 p.m. \ No newline at end of file diff --git a/data/housing/housing_departments.txt b/data/housing/housing_departments.txt new file mode 100644 index 0000000..ff42734 --- /dev/null +++ b/data/housing/housing_departments.txt @@ -0,0 +1,24 @@ +URL: https://housing.utdallas.edu/housing-departments +Title: Housing Departments | UTD +Depth: 1 + +------------------------------------------------------------ + +Housing Departments +Residential Life + +UT Dallas views residential life as an integral part of the education process. Residential Life seeks to provide structure, support and a sense of community that will complement your classroom experience. + +Living Learning Communities + +Living Learning Communities (LLCs) are groups of students who share academic goals and interests. LLC members live in designated residence halls or apartments, enroll in community-specific courses and participate in group activities designed to enhance learning. LLC students typically develop peer relationships more quickly, enjoy increased faculty interaction outside the classroom and maintain higher GPAs. + +Assessment and Guest Housing + +The University of Texas at Dallas welcomes interns and researchers traveling to the Dallas Fort-Worth area to live on a safe, modern and amenity-filled campus each summer. With multiple floorplans available, UT Dallas offers competitive pricing for the metroplex and its location in Richardson, Texas, is minutes away from Texas Instruments, Raytheon, State Farm and many more large corporations.  + +Additional information available at Corporate Intern and Summer Guest Housing + +Housing Operations + +The office of the associate vice president for Student Affairs oversees all housing operations on campus, including all residence halls and apartments. \ No newline at end of file diff --git a/data/housing/housing_departments_corporate_intern_and_summer_guest_housing.txt b/data/housing/housing_departments_corporate_intern_and_summer_guest_housing.txt new file mode 100644 index 0000000..3aafc1a --- /dev/null +++ b/data/housing/housing_departments_corporate_intern_and_summer_guest_housing.txt @@ -0,0 +1,38 @@ +URL: https://housing.utdallas.edu/housing-departments/corporate-intern-and-summer-guest-housing +Title: Corporate Intern and Summer Guest Housing | UTD +Depth: 2 + +------------------------------------------------------------ + +Corporate Intern and Guest Housing +Housing Departments > Assessment and Guest Housing +APPLY OR SIGN IN +Related Pages +Housing Options +Resources +Amenities +Policies +Pricing +What to Bring +FAQ + +The University of Texas at Dallas welcomes interns and researchers traveling to the Dallas Fort-Worth area to live on a safe, modern and amenity-filled campus each summer. With multiple floorplans available, UT Dallas offers competitive pricing for the metroplex and its location in Richardson, Texas, is minutes away from Texas Instruments, Raytheon, State Farm and many more large corporations. + +We encourage anyone interested in applying to do so following a review of our pricing and policies pages. Any questions about the program or our offerings should be sent to housing@utdallas.edu. + +2026 Availability + +The 2026 Corporate Intern Application will open on Monday, Jan. 12, 2026 at 10 a.m. Guest Housing for Summer 2026 will be available from Jun. 1 – Aug. 2, 2026. + +Member Of +Timeline +Jan. 12, 2026: 2026 Corporate Intern Application opens. To select a preferred roommate, all preferred roommates must email housing@utdallas.edu and include the following information: +Name(s) of all preferred roommates. +A written request to be paired with the names listed in the email. +Apr. 14, 2026: 2026 Corporate Intern Application closes. +Apr. 15, 2026: Final day to cancel your 2026 Corporate Intern reservation. Interns must cancel their reservation within the application portal and also submit a formal written request to housing@utdallas.edu. Cancellation for any reason after Apr. 15, 2026 will result in forfeiture of all payments received. +Apr. 17, 2026: Intern Offers sent. Applicants will receive an email with their assigned room space and the date by which they must accept or decline the offer. +Apr. 30, 2026: Check-In and Move-In instructions will be sent to all interns who have accepted their offer. +June. 1, 2026: 2026 Corporate Intern Move-In. +July 1-15. 2026: 2026 Corporate Intern 2025 Move-Out instructions will be sent to all interns. +Aug. 2, 2026: 2026 Corporate Intern Move-Out day. All interns, unless given special permission, are required to be moved out by 7 p.m. on Aug. 2, 2026. \ No newline at end of file diff --git a/data/housing/housing_departments_guest_housing.txt b/data/housing/housing_departments_guest_housing.txt new file mode 100644 index 0000000..52af384 --- /dev/null +++ b/data/housing/housing_departments_guest_housing.txt @@ -0,0 +1,38 @@ +URL: https://housing.utdallas.edu/housing-departments/guest-housing +Title: Corporate Intern and Summer Guest Housing | UTD +Depth: 1 + +------------------------------------------------------------ + +Corporate Intern and Guest Housing +Housing Departments > Assessment and Guest Housing +APPLY OR SIGN IN +Related Pages +Housing Options +Resources +Amenities +Policies +Pricing +What to Bring +FAQ + +The University of Texas at Dallas welcomes interns and researchers traveling to the Dallas Fort-Worth area to live on a safe, modern and amenity-filled campus each summer. With multiple floorplans available, UT Dallas offers competitive pricing for the metroplex and its location in Richardson, Texas, is minutes away from Texas Instruments, Raytheon, State Farm and many more large corporations. + +We encourage anyone interested in applying to do so following a review of our pricing and policies pages. Any questions about the program or our offerings should be sent to housing@utdallas.edu. + +2026 Availability + +The 2026 Corporate Intern Application will open on Monday, Jan. 12, 2026 at 10 a.m. Guest Housing for Summer 2026 will be available from Jun. 1 – Aug. 2, 2026. + +Member Of +Timeline +Jan. 12, 2026: 2026 Corporate Intern Application opens. To select a preferred roommate, all preferred roommates must email housing@utdallas.edu and include the following information: +Name(s) of all preferred roommates. +A written request to be paired with the names listed in the email. +Apr. 14, 2026: 2026 Corporate Intern Application closes. +Apr. 15, 2026: Final day to cancel your 2026 Corporate Intern reservation. Interns must cancel their reservation within the application portal and also submit a formal written request to housing@utdallas.edu. Cancellation for any reason after Apr. 15, 2026 will result in forfeiture of all payments received. +Apr. 17, 2026: Intern Offers sent. Applicants will receive an email with their assigned room space and the date by which they must accept or decline the offer. +Apr. 30, 2026: Check-In and Move-In instructions will be sent to all interns who have accepted their offer. +June. 1, 2026: 2026 Corporate Intern Move-In. +July 1-15. 2026: 2026 Corporate Intern 2025 Move-Out instructions will be sent to all interns. +Aug. 2, 2026: 2026 Corporate Intern Move-Out day. All interns, unless given special permission, are required to be moved out by 7 p.m. on Aug. 2, 2026. \ No newline at end of file diff --git a/data/housing/housing_departments_guest_housing_amenities_guest_housing.txt b/data/housing/housing_departments_guest_housing_amenities_guest_housing.txt new file mode 100644 index 0000000..9bb1ead --- /dev/null +++ b/data/housing/housing_departments_guest_housing_amenities_guest_housing.txt @@ -0,0 +1,85 @@ +URL: https://housing.utdallas.edu/housing-departments/guest-housing/amenities-guest-housing +Title: Amenities: Guest Housing | UTD +Depth: 2 + +------------------------------------------------------------ + +Amenities: Guest Housing +Housing Departments > Assessment and Guest Housing > Amenities: Guest Housing +APPLY OR SIGN IN +Related Pages +Housing Options +Resources +Amenities +Policies +Pricing +What to Bring +FAQ +General Housing Amenities +Wireless & wired internet +Community laundry centers +Central air conditioning & heat +Card access to buildings and suite/apartment front doors +24-hour on-call staff & lockout services +All utilities provided at no extra cost +Study lounges +On-site maintenance staff +Outdoor gathering areas and fire pit +Emergency call stations +24-hour police patrol +University Commons +Canyon Creek Heights +Recreation: Activity Center +Community-Specific Amenities +University Commons Amenities + +Virtual Tour + +Private bedrooms lockable with key. + +Bedrooms furnished with loftable Twin XL bed & mattress, desk, desk chair, and dresser. + +Common living area furnished with a couch. + +Community kitchen and laundry room on the first floor. + +Shared bathroom with separate shower and toilet. Shared vanity with three sinks. + +Built in counters. + +Exterior window with mini blinds. + +Canyon Creek Heights Amenities + +Virtual Tour + +Private bedrooms lockable with key. + +Bedrooms furnished with bed & mattress, desk, desk chair, and dresser. + +Common living area furnished with a couch, coffee table, and barstools. + +Fully equipped kitchen with full-sized refrigerator & freezer with ice maker, electric stove & oven, microwave, dishwasher, and garbage disposal. + +Private and semi-private bathrooms. + +Pantry and storage space. + +Exterior window with mini blinds. + +Outdoor Pools +Campus Recreation +Outdoor swimming pools +Volleyball, basketball, and tennis courts +Ping pong, foosball, and pool tables in the residence halls +Walking paths +Bicycle friendly neighborhoods +Access to Rec Center West and the Activity Center with a natatorium, fitness center, indoor courts, climbing wall, and athletic fields is available for purchase at the time of your housing registration. Group use of any available facility requires a reservation and is subject to housing and university rules. +Dining Options +Dining and Food Services +Dining hall near residence hall housing areas with all-you-care-to-eat menu. +Banquet facilities for up to 300 diners. +Retail options in the central part of campus featuring Chick-fil-A, Firehouse Subs, The Halal Shack, Einstein Bros Bagels, Buffalo Wild Wings, and more. +Catering available. + +For retail locations and hours, visit UT Dallas Dining Services. \ No newline at end of file diff --git a/data/housing/housing_departments_guest_housing_guest_housing_resources.txt b/data/housing/housing_departments_guest_housing_guest_housing_resources.txt new file mode 100644 index 0000000..3adca14 --- /dev/null +++ b/data/housing/housing_departments_guest_housing_guest_housing_resources.txt @@ -0,0 +1,32 @@ +URL: https://housing.utdallas.edu/housing-departments/guest-housing/guest-housing-resources +Title: Guest Housing Resources | UTD +Depth: 2 + +------------------------------------------------------------ + +Guest Housing Resources +Housing Departments > Assessment and Guest Housing > Guest Housing Resources +APPLY OR SIGN IN +Related Pages +Housing Options +Resources +Amenities +Policies +Pricing +What to Bring +FAQ +Amenities + +View guest housing amenities with links to virtual tours, information about recreational facilities, dining and food services. + +Policies + +Review guest housing policies covering guest housing requirements, application procedures, insurance and required forms. + +What to Bring + +View lists of items that should be brought for your stay, disallowed items and optional recommended items. + +FAQs + +View a list of common questions and answers about guest housing. \ No newline at end of file diff --git a/data/housing/housing_departments_guest_housing_housing_options_guest.txt b/data/housing/housing_departments_guest_housing_housing_options_guest.txt new file mode 100644 index 0000000..b0165f4 --- /dev/null +++ b/data/housing/housing_departments_guest_housing_housing_options_guest.txt @@ -0,0 +1,32 @@ +URL: https://housing.utdallas.edu/housing-departments/guest-housing/housing-options-guest +Title: Housing Options: Interns & Summer Guests - University Housing | The University of Texas at Dallas +Depth: 2 + +------------------------------------------------------------ + +Housing Options: Interns & Summer Guests +Housing Departments > Assessment and Guest Housing > Housing Options: Interns & Summer Guests +APPLY OR SIGN IN +Related Pages +Housing Options +Resources +Amenities +Policies +Pricing +What to Bring +FAQ +Contact + +housing@utdallas.edu + +Corporate interns and summer guests have two housing options: University Commons (our most centrally-located and highly-requested community), or Canyon Creek Heights. + +University Commons +University Commons – Furnished Residence Halls + +University Commons residence halls feature three-bedroom suites, recreation areas, large communal spaces and study lounges. + +Canyon Creek Heights +Canyon Creek Heights – Furnished Apartments + +Canyon Creek Heights apartments offer a variety of floor plans and amenities. \ No newline at end of file diff --git a/data/housing/housing_departments_guest_housing_policies_summer_guest_housing.txt b/data/housing/housing_departments_guest_housing_policies_summer_guest_housing.txt new file mode 100644 index 0000000..645b5ed --- /dev/null +++ b/data/housing/housing_departments_guest_housing_policies_summer_guest_housing.txt @@ -0,0 +1,61 @@ +URL: https://housing.utdallas.edu/housing-departments/guest-housing/policies-summer-guest-housing +Title: Policies: Summer Guest Housing - University Housing | The University of Texas at Dallas +Depth: 2 + +------------------------------------------------------------ + +Policies: Summer Guest Housing +Housing Departments > Assessment and Guest Housing > Policies: Summer Guest Housing +APPLY OR SIGN IN +Related Pages +Housing Options +Resources +Amenities +Policies +Pricing +What to Bring +FAQ +Summer Guest Housing Requirements +Interns must be 18 years of age or older. +You do not have to be a UT Dallas student to live in summer guest housing. Any person with an internship, research affiliation, or who is taking summer courses in the Dallas-Fort Worth area is eligible to apply. An affiliation with UT Dallas is not required. +All interns must provide proof of internship from the sponsoring company, such as a letter of verification from a supervisor. +All summer guests must comply with the Resident Handbook. +Full payment must be received prior to move in. Guest Housing will provide an invoice based on the number of nights stay. +Insurance + +Summer interns are required to provide information to be added to a comprehensive general liability and property damage insurance policy for UT Dallas. This policy does not cover injury to the individual or damage to the individual’s property. We encourage you to purchase personal renters insurance and health insurance to cover yourself. + +Process Overview + +APPLY + +Submit an application + +CONFIRM + +You will receive an email acknowledgement once your application has been processed by our office. + +OFFER + +Applicants will receive an email notifying them that a summer room assignment has been offered. + +ACCEPT OR DECLINE + +All guests will have the ability to accept or decline their offer within the guest housing portal. + +PAY + +Payment of the balance on the guest housing portal is due in full prior to move-in. Payment received after April 15 is non-refundable. + +READ EMAILS + +Email(s) will be sent prior to the date of arrival detailing important information and instructions. + +NEED TO +CANCEL? + +You may cancel your application at any time via the housing portal or via housing@utdallas.edu + +CHECK IN + +You will be provided with detailed instructions on how to check-in on campus prior to your arrival date. \ No newline at end of file diff --git a/data/housing/housing_departments_guest_housing_pricing_summer_guest_housing.txt b/data/housing/housing_departments_guest_housing_pricing_summer_guest_housing.txt new file mode 100644 index 0000000..48eec0b --- /dev/null +++ b/data/housing/housing_departments_guest_housing_pricing_summer_guest_housing.txt @@ -0,0 +1,97 @@ +URL: https://housing.utdallas.edu/housing-departments/guest-housing/pricing-summer-guest-housing +Title: Pricing: Summer Guest Housing - University Housing | The University of Texas at Dallas +Depth: 2 + +------------------------------------------------------------ + +Pricing: Summer Guest Housing +Housing Departments > Assessment and Guest Housing > Pricing: Summer Guest Housing +APPLY OR SIGN IN +Related Pages +Housing Options +Resources +Amenities +Policies +Pricing +What to Bring +FAQ + Pricing Calculator: Summer Guest Housing +Begin a cost estimation +University Commons Residence Halls + +Private bedroom in a three-bedroom suite. + +30 days or less: $50/day per person + $300 service fee + +31+ days: $3,050 total flat fee per person – due prior to check in + +One microwave and refrigerator combo included per suite. Community kitchen is available in the lobby. + +Photos of Model Suite + +Canyon Creek Heights Apartments + +Private bedroom in a four-bedroom apartment with kitchen. + +30 days or less: $52/day per person + $300 service fee +31+ days: $3,172 total per person – due prior to check in + +Photos: Model Apartment + +Private bedroom in a two-bedroom apartment with kitchen. + +30 days or less: $55/day per person + $300 service fee +31+ days: $3,355 total per person – due prior to check in + +Photos: Model Apartment + +Private one-bedroom apartment with kitchen. + +30 days or less: $65/day per person + $300 service fee +31+ days: $3,965 total per person – due prior to check in +These spaces are in high demand, and limited spaces are available. + +Photos: Model Apartment + +Add Ons +Meal Plans + +Meal Money can be used in any University dining location open during the summer on campus. Meals can be added at any time during your stay. + +Summer 80 | $971.19 (includes tax) + +80 meals at Dining Hall West +This plan does not include Meal Money + +Summer 60 | $794.95 (includes tax) + +60 meals at Dining Hall West +$50 in Meal Money + +Summer 30 | $685.55 (includes tax) + +30 meals at Dining Hall West +$300 in Meal Money + +Dining Hall walk-in pricing: + +Breakfast: $9.65 (does not include tax) +Lunch: $11.77 (does not include tax) +Dinner: $13.26 (does not include tax) +Activity Center + +An Activity Center membership is good for your entire stay. + +$100/person +Parking + +Parking permits are purchased directly from the Parking & Transportation Office. Email park@utdallas.edu with further questions. + +There is a seven night minimum requirement for all accommodations. +A $300 service charge will be added to any stay totaling 30 days or less. +Payment is due in full in advance or at check in. Payment may be made by check, money order, or credit card. +Cancellations made by April 15 will receive a full refund of the housing charges paid in advance. Cancellations made for any reason after April 15 will result in forfeiture of all payments received. + +No refunds will be provided after occupancy is taken, including for unused meals, parking, and Activity Center membership. +Rooms will be checked after departure. A final bill may be issued for any damages incurred. +Linens are not included. Please familiarize yourself with what to bring. \ No newline at end of file diff --git a/data/housing/housing_departments_guest_housing_what_to_bring_summer_guest_housing.txt b/data/housing/housing_departments_guest_housing_what_to_bring_summer_guest_housing.txt new file mode 100644 index 0000000..05fe081 --- /dev/null +++ b/data/housing/housing_departments_guest_housing_what_to_bring_summer_guest_housing.txt @@ -0,0 +1,46 @@ +URL: https://housing.utdallas.edu/housing-departments/guest-housing/what-to-bring-summer-guest-housing +Title: What To Bring: Summer Guest Housing | UTD +Depth: 2 + +------------------------------------------------------------ + +What To Bring: Summer Guest Housing +Housing Departments > Assessment and Guest Housing > What To Bring: Summer Guest Housing +APPLY OR SIGN IN +Related Pages +Housing Options +Resources +Amenities +Policies +Pricing +What to Bring +FAQ +University Commons +Items To Bring +In University Commons, mattresses are Twin XL. +In Canyon Creek Heights North, mattresses are Twin XL. +In Canyon Creek Heights South, mattresses are full-sized. +Bed linens. +Pillow. +Towels and washcloths. +Toiletries (soap, shampoo, hand soap, deodorant, toothbrush, toothpaste, etc.). +Blow dryer. +Laundry detergent. +Laundry bag. +Trash bags. +Cleaning supplies. +Clothes hangers. +Toilet paper. +Swimsuit. (There are pools, but no lifeguards on duty.) +Basketball or other sports gear. +Coffee pot with automatic shut off. +Canyon Creek Heights +Items Not to Bring +Candles. +Toasters. +Hot plates. +Weapons. +Drugs/illegal substances. +Pets. +Tobacco. UT Dallas is a tobacco free campus . +For a full list of prohibited items, see the Resident Handbook. \ No newline at end of file diff --git a/data/housing/housing_options.txt b/data/housing/housing_options.txt new file mode 100644 index 0000000..ff8b0bd --- /dev/null +++ b/data/housing/housing_options.txt @@ -0,0 +1,26 @@ +URL: https://housing.utdallas.edu/housing-options +Title: Housing Options | UTD +Depth: 1 + +------------------------------------------------------------ + +Housing Options +University Commons + +Residence Halls + +University Commons residence halls feature three-bedroom suites, recreation areas, community lounges, hangout areas, and study spaces. Available to first-year students*. + +University Village + +Unfurnished Apartments + +University Village apartments offer a variety of floor plans and amenities. Available to non-first year students. + +Canyon Creek Heights + +Furnished Apartments + +Canyon Creek Heights apartments offer a variety of floor plans and amenities. Available to non-first year students. + +* First-year students are those who graduate high school in May or June prior to their first semester at UT Dallas. \ No newline at end of file diff --git a/data/housing/housing_options_canyon_creek_heights.txt b/data/housing/housing_options_canyon_creek_heights.txt new file mode 100644 index 0000000..a3450dd --- /dev/null +++ b/data/housing/housing_options_canyon_creek_heights.txt @@ -0,0 +1,57 @@ +URL: https://housing.utdallas.edu/housing-options/canyon-creek-heights +Title: Canyon Creek Heights | Housing | UTD +Depth: 1 + +------------------------------------------------------------ + +Canyon Creek Heights +1 +2 +3 +4 +5 +6 +Previous +Next + +Canyon Creek Heights apartments offer students a variety of fully furnished floor plans and the comfort of home within walking distance to class. + +These buildings are available to sophomore, junior, senior and graduate students. If you are a first-year student, see the first-year housing page. + + View the location of the apartments on the campus map. + +PRICING & FLOOR PLANS +VIRTUAL TOUR +MODEL APARTMENT PHOTOS +RESIDENT HANDBOOK +ROOM AMENITIES +Furnished with bed, desk, dresser, chair, sofa, table. +Refrigerator/freezer with ice maker. +Dishwasher. +Electric stove/oven. +Garbage disposal. +Microwave. +Central heat/air conditioning. +Wireless & wired internet. +All utilities provided at no additional cost. +COMMUNITY AMENITIES +Peer Advisors +Study nooks, meeting room and social lobby. +Wireless & wired internet. +Outdoor gathering area. +Access to all University Housing amenities: swimming pools, volleyball courts, outdoor basketball and tennis courts. +Landscaped and maintained grounds. +Bicycle racks and repair stands. +On-site maintenance staff. +Mailbox stations. +24-hour campus police patrol. +24-hour lockout service. +Emergency call stations. + +Housing Agreement Information + +The University of Texas at Dallas provides housing on an individual contracted basis. We offer housing options based on one bed per student, and we do not allow a student to reserve more than one bedroom except in limited cases for family housing, if available. If a student wishes to live alone, a student may select from the available 1-bedroom unit options on campus. + +Meal Plans + +Students may choose to purchase one of a variety of optional meal plans. Learn more about on campus dining. \ No newline at end of file diff --git a/data/housing/housing_options_housing_availability.txt b/data/housing/housing_options_housing_availability.txt new file mode 100644 index 0000000..1a70e87 --- /dev/null +++ b/data/housing/housing_options_housing_availability.txt @@ -0,0 +1,139 @@ +URL: https://housing.utdallas.edu/housing-options/housing-availability +Title: Housing Availability | UTD +Depth: 1 + +------------------------------------------------------------ + +Housing Availability +Spring 2026 Non-First Year Housing Availability +Canyon Creeks Heights + +Furnished apartments with bed, desk, dresser, chair, sofa and table. + +Canyon Creek Heights North 2×2 + +Cost/Semester* + +$4,575 + +Availability: + +Not Available + +Canyon Creek Heights North 4×2 + +Cost/Semester* + +$4,075 + +Availability: + +Not Available + +Canyon Creek Heights South 1×1 + +Cost/Semester* + +$5,450 + +Availability: + +Not Available + +Canyon Creek Heights South 2×2 + +Cost/Semester* + +$4,625 + +Availability: + +Not Available + +* Rates are subject to change. Price includes all utilities. + +University Village + +Non-furnished apartments. + +University Village Efficiency + +Cost/Semester* + +$4,338 + +Availability: + +Not Available + +University Village 1×1 A + +Cost/Semester* + +$4,725 + +Availability: + +Not Available + +University Village 1×1 B + +Cost/Semester* + +$5,150 + +Availability: + +Not Available + +University Village 1×1 C + +Cost/Semester* + +$5,150 + +Availability: + +Not Available + +University Village 1×1 D + +Cost/Semester* + +$5,325 + +Availability: + +Not Available + +University Village 2×2 A + +Cost/Semester* + +$3,825 + +Availability: + +Not Available + +University Village 2×2 B + +Cost/Semester* + +$4,275 + +Availability: + +Available + +University Village 4×2 + +Cost/Semester* + +$3,250 + +Availability: + +Available + +* Rates are subject to change. Price includes all utilities. \ No newline at end of file diff --git a/data/housing/housing_options_university_commons.txt b/data/housing/housing_options_university_commons.txt new file mode 100644 index 0000000..a3d9094 --- /dev/null +++ b/data/housing/housing_options_university_commons.txt @@ -0,0 +1,67 @@ +URL: https://housing.utdallas.edu/housing-options/university-commons +Title: University Commons | Housing | UTD +Depth: 1 + +------------------------------------------------------------ + +University Commons +1 +2 +3 +4 +5 +6 +7 +Previous +Next + +University Commons residence halls feature fully furnished three-bedroom suites, a variety of recreation areas, community lounges, hangout areas, and study spaces. + +All first-year students who choose to live on campus are required to live in University Commons. + + View the location of the residence halls and housing office on the campus map. + +PRICING & FLOOR PLANS +VIRTUAL TOUR +MODEL SUITE PHOTOS +RESIDENT HANDBOOK +ROOM AMENITIES +Furnished with loftable bed & mattress (Twin XL), desk, chair and dresser. +Exterior window with mini blinds. +Lockable with key entry. +Wireless & wired internet. +All utilities provided at no additional cost. +COMMON LIVING AREAS +Shared suite entry with student ID card access. +Living area with couch. +Area for small refrigerator, microwave and TV. See photo with measurements. +Built-in counters. +Shared bathroom with separate shower and toilet. +Shared vanity includes three sinks. +COMMUNITY AMENITIES +Peer Advisors. +Study rooms, study nooks and social lobby. +Outdoor gathering areas. +Access to all University Housing amenities: swimming pools, volleyball courts, outdoor basketball and tennis courts. +Wireless & wired internet.  +Laundry room in each building. +Public kitchen in each building. +Table tennis, billiards and foos ball equipment. +Landscaped and maintained grounds. +Bicycle racks and repair stands. +On-site maintenance staff. +Indoor mailbox stations. +24-hour campus police patrol. +24-hour lockout service. +Emergency call stations. +Housing Agreement Information + +The University of Texas at Dallas provides housing on an individual contracted basis. We offer housing options based on one bed per student, and we do not allow a student to reserve more than one bedroom except in limited cases for family housing, if available. + +Living Learning Communities + +Residents are invited to apply to be part of our Living Learning Communities (LLCs), where first-year students with similar majors and interests live together, study together, participate in social functions and have the opportunity to interact with faculty. Participation in LLCs is limited, and communities are filled on a first-come, first-served basis. LLC acceptance takes priority over roommate requests. + +Meal Plans + +University Commons residents are required to purchase a meal plan. Learn more about on campus dining. \ No newline at end of file diff --git a/data/housing/housing_options_university_village.txt b/data/housing/housing_options_university_village.txt new file mode 100644 index 0000000..5babeb2 --- /dev/null +++ b/data/housing/housing_options_university_village.txt @@ -0,0 +1,71 @@ +URL: https://housing.utdallas.edu/housing-options/university-village +Title: University Village | Housing | UTD +Depth: 1 + +------------------------------------------------------------ + +University Village +1 +2 +3 +4 +5 +6 +Previous +Next + +University Village apartments offer students a variety of unfurnished floor plans and the comfort of home within walking distance to class. + +If you are a sophomore, junior, senior or graduate student, you will find housing information here. If you are a first-year student, see the First-Year Housing page. + +View the locations of the University Village buildings and housing office on a map. + +PRICING & FLOOR PLANS +VIRTUAL TOUR +MODEL SUITE PHOTOS +RESIDENT HANDBOOK +ROOM AMENITIES +Refrigerator/freezer with ice maker. +Dishwasher*. +Electric stove/oven. +Garbage disposal. +Microwave. +Wireless & wired internet. +Mini blinds. +Washer/dryer*. +Central heat/air conditioning. +Vinyl floors (kitchen/bathroom). +All utilities provided at no additional cost. + +*In select units + +COMMUNITY AMENITIES +Peer Advisors. +Study centers, meeting rooms and lounges. +Wireless & wired internet. +Outdoor grills. +Swimming pools. +Sand volleyball courts. +Outdoor basketball and tennis courts. +Billiards and ping-pong tables. +Landscaped and maintained grounds. +Picnic areas. +Bicycle racks and repair stands. +On-site maintenance staff. +Mailbox stations. +24-hour campus police patrol. +24-hour lockout service. +Emergency call stations. +Housing Agreement Information + +The University of Texas at Dallas provides housing on an individual contracted basis. We offer housing options based on one bed per student, and we do not allow a student to reserve more than one bedroom except in limited cases for family housing, if available. If a student wishes to live alone, a student may select from the available 1-bedroom unit options on campus. + +Meal Plans + +Students may choose to purchase one of a variety of optional meal plans. Learn more about on campus dining. + +Family Housing + +Please inquire with University Housing for available family housing options. There are limited facilities within University Housing that are available for students with spouses and children. + +These spaces are offered on a first-come, first-served basis. We cannot guarantee family housing options will be available. All persons residing in a family housing unit are required to sign an apartment housing agreement. Canyon Creek Heights and University Commons are single bedroom housing options and are not available as family housing. \ No newline at end of file diff --git a/data/housing/housing_staff.txt b/data/housing/housing_staff.txt new file mode 100644 index 0000000..85ed065 --- /dev/null +++ b/data/housing/housing_staff.txt @@ -0,0 +1,604 @@ +URL: https://housing.utdallas.edu/housing-staff +Title: Housing Staff | UTD +Depth: 1 + +------------------------------------------------------------ + +Housing Staff +Housing Offices +Housing Leadership +Housing Staff +University Housing Departments +Show All +Leadership +Residential Life +Housing Operations +Housing Administration +Living Learning Communities +Business Services +Assessment and Guest Housing +Search for staff member +Leadership +Matthew Grief + +Associate Vice President for Student Affairs +SSA 14.120G +972-883-6232 +grief@utdallas.edu + +Ryan White + +Assistant Vice President of Residential Life +Andromeda 1.405 +972-883-7430 +ryan.white@utdallas.edu + +Hayley Minton + +Senior Director of Housing Operations +UVHO 1.830A +972-883-7300 +hminton@utdallas.edu + +Stan Scroggins + +Senior Director of Business Services +Helix 2.005 +972-883-7313 +stan.scroggins@utdallas.edu + +Kaytie Harwood + +Director of Residential Life +Andromeda 1.407 +972-883-7430 +kaytie.harwood@utdallas.edu + +Sarah Mirarchi + +Associate Director of Housing Administration +Andromeda 1.414 +972-883-7412 +sarah.mirarchi@utdallas.edu + +Taylor Lawson + +Associate Director of Housing Operations +UVHO 1.875 +972-883-7300 +taylor.lawson@utdallas.edu + +Andres DeJesus + +Associate Director of Housing Operations +HSW 1.225 +972-883-7300 +andres.dejesus@utdallas.edu + +Elizabeth Johnson + +Associate Director of Assessment and Guest Housing +Andromeda 1.402 +972-883-7427 +elizabeth.johnson@utdallas.edu + +Seth Harwood + +Program Manager for Risk Management +UVHO 1.853 +972-883-7300 +seth.harwood@utdallas.edu + +Christina Schwiderski + +Associate Director of Residential Education +Andromeda 1.415 +972-883-7430 +christina.schwiderski@utdallas.edu + +Residential Life +Matthew Grief + +Associate Vice President for Student Affairs +SSA 14.120G +972-883-6232 +grief@utdallas.edu + +Ryan White + +Assistant Vice President of Residential Life +Andromeda 1.405 +972-883-7430 +ryan.white@utdallas.edu + +Kaytie Harwood + +Director of Residential Life +Andromeda 1.407 +972-883-7430 +kaytie.harwood@utdallas.edu + +Christina Schwiderski + +Associate Director of Residential Education +Andromeda 1.415 +972-883-7430 +christina.schwiderski@utdallas.edu + +Lisa Koontz + +Assistant Director of Apartment Area +Andromeda 1.413 +972-883-7430 +lisa.koontz@utdallas.edu + +Luis Robles + +Assistant Director of University Commons +Andromeda 1.411 +972-883-7430 +luis.robles@utdallas.edu + +Maeve Benson + +Residential Life Coordinator +Capella Hall + +Capella 1.006 +972-883-7429 +mbenson@utdallas.edu + +ME Lasso de la Vega + +Residential Life Coordinator +Andromeda Hall +Andromeda 1.003 +972-883-7424 +maria.emilia@utdallas.edu + +Victoria Nord + +Residential Life Coordinator +Buildings 2-31 & Canyon Creek Heights +CCHS 1.118 + +972-883-7332 +victoria.nord@utdallas.edu + +Alexis Rodriguez + +Residential Life Coordinator +Buildings 33-51 +Phase 8 Office +972-883-7423 +alexis.rodriguez3@utdallas.edu + +Ren Lafleur + +Assistant Residential Life Coordinator +Vega Hall +Vega 1.006 +972-883-7421 +ren.lafleur@utdallas.edu + +Tanya Lorenzana + +Assistant Residential Life Coordinator +Sirius Hall +Sirius 1.007 +972-883-7333 +tanya.lorenzana@utdallas.edu + +Jordan Lee + +Assistant Residential Life Coordinator +Helix Hall +Helix 1.006 +972-883-7426 +jordan.lee@utdallas.edu + +Kayla Fairclough + +Assistant Residential Life Coordinator +Canyon Creek Heights North + +CCHN 1.152 +972-883-7337 +kayla.fairclough@utdallas.edu + +Baselios Girmai + +Assistant Residential Life Coordinator +Canyon Creek Heights South + +CCHS 1.122 +972-883-5348 +baselios.girmai@utdallas.edu + +Andy Guzman Barrera + +Assistant Residential Life Coordinator +Buildings 53-59 + +Phase 8 Office +972-883-7281 +andy.guzmanbarrera@utdallas.edu + +Ethan Nelson + +Assistant Residential Life Coordinator +Buildings 61-67 + +Phase 8 Office +972-883-7321 +ethan.nelson@utdallas.edu + +Alecia Martin + +Assistant Coordinator of Residential Education +Helix 1.007 +972-883-7363 +alecia.martin@utdallas.edu + +LaKendra Herring + +Assistant Coordinator of Residential Education +Vega 1.007 +972-883-3477 +lakendra.herring@utdallas.edu + +Angela Gutierrez + +Administrative Assistant II +UVHO +972-883-6358 +angela.gutierrez2@utdallas.edu + +Housing Operations +Matthew Grief + +Associate Vice President for Student Affairs +SSA 14.120G +972-883-6232 +grief@utdallas.edu + +Hayley Minton + +Senior Director of Housing Operations +UVHO 1.830A +972-883-7300 +hminton@utdallas.edu + +Taylor Lawson + +Associate Director of Housing Operations +UVHO 1.875 +972-883-7300 +taylor.lawson@utdallas.edu + +Andres DeJesus + +Associate Director of Housing Operations +HSW 1.225 +972-883-7300 +andres.dejesus@utdallas.edu + +Seth Harwood + +Program Manager for Risk Management +UVHO 1.853 +972-883-7300 +seth.harwood@utdallas.edu + +Ashleigh Beckmann + +Assistant Director of Housing Operations +UVHO 1.860 +972-883-7306 +ashleigh.beckmann@utdallas.edu + +Jackson Havis + +Assistant Superintendent +of Housing Operations +UC Maintenance +972-883-7300 +jackson.havis@utdallas.edu + +Michael Crocker + +Assistant Superintendent of Housing Operations +CCHS 1.114 +972-883-7300 +michael.crocker@utdallas.edu + +Cesar Arreola + +Assistant Superintendent +of Housing Operations +UVHO 1.805 +972-883-7300 +cesar.arreola@utdallas.edu + +Maria Mares + +Building Services Supervisor +HSW 1.210 +972-883-7300 +maria.mares@utdallas.edu + +Leonard Griffin + +Inventory Control Specialist +HSW 1.130 +972-883-7300 +lxg132230@utdallas.edu + +Jessika Howard + +Housing Operations Coordinator +UVHO 1.839 +972-883-7334 +jessika.howard@utdallas.edu + +Megan Pruitt + +Housing Operations Coordinator +Andromeda 1.004 +972-883-7315 +megan.pruitt@utdallas.edu + +Sarah Alexander + +Administrative Project Coordinator +UVHO 1.865 +972-883-6282 +sarah.alexander@utdallas.edu + +Linda McCombs + +Administrative Project Coordinator +UVHO 1.845 +972-883-7364 +linda.mccombs@utdallas.edu + +Alex Guyton + +Assistant Housing Operations Coordinator +Andromeda 1.001 +972-883-5564 +alex.guyton@utdallas.edu + +Maria Rojas-Caballero + +Assistant Housing Operations Coordinator +Sirius 1.003 +972-883-7331 +maria.rojas-caballero@utdallas.edu + +Cheryl Harris + +Assistant Housing Operations Coordinator +CCHN 1.148 +972-883-7316 +cheryl.harris@utdallas.edu + +Quindon Jefferson-Bowers + +Assistant Housing Operations Coordinator +UVHO 1.830 C +972-883-7357 +quindon.jefferson-bowers@utdallas.edu + +Caitlin DeLeon + +Front Desk Assistant +caitlin.deleon@utdallas.edu + +Kim Phillips + +Front Desk Assistant +kimberly.phillips@utdallas.edu + +Kayla Peeples + +Front Desk Assistant +kayla.peeples@utdallas.edu + +Olivia Dean + +Front Desk Assistant +zane.dean@utdallas.edu + +Ruth Bombio + +Front Desk Assistant +ruth.bombio@utdallas.edu + +Heather Goodrum + +Front Desk Assistant +heather.goodrum@utdallas.edu + +Sarby Drolet + +Front Desk Assistant +sarby.drolet@utdallas.edu + +Harrison Holzer + +Front Desk Assistant +harrison.holzer@utdallas.edu + +Nate Tvete + +Front Desk Assistant +nathaniel.tvete@utdallas.edu + +Angela Gutierrez + +Administrative Assistant II +UVHO +972-883-6358 +angela.gutierrez2@utdallas.edu + +Housing Administration +Matthew Grief + +Associate Vice President for Student Affairs +SSA 14.120G +972-883-6232 +grief@utdallas.edu + +Ryan White + +Assistant Vice President of Residential Life +Andromeda 1.405 +972-883-7430 +ryan.white@utdallas.edu + +Sarah Mirarchi + +Associate Director of Housing Administration +Andromeda 1.414 +972-883-7412 +sarah.mirarchi@utdallas.edu + +Emily Ross + +Coordinator of Housing Administration +Andromeda 1.408 +972-883-7413 +emily.ross@utdallas.edu + +Emily Rigsby + +Coordinator of Housing Administration +Andromeda 1.410 +972-883-7430 +emily.rigsby@utdallas.edu + +Vacant + +Coordinator of Housing Administration +Andromeda 1.408 + +Living Learning Communities +Matthew Grief + +Associate Vice President for Student Affairs +SSA 14.120G +972-883-6232 +grief@utdallas.edu + +Ryan White + +Assistant Vice President of Residential Life +Andromeda 1.405 +972-883-7430 +ryan.white@utdallas.edu + +Christina Schwiderski + +Associate Director of Residential Education +Andromeda 1.415 +972-883-7430 +christina.schwiderski@utdallas.edu + +Mariessa Joseph + +Assistant Director of Living Learning Communities +Andromeda +972-883-7430 +mariessa.joseph@utdallas.edu + +Madison Polk + +Coordinator +Vega 2.004 +madison.polk@utdallas.edu + +Nicky Dang + +Coordinator +Vega 2.007 +972-883-7456 +nicky.dang@utdallas.edu + +Jennie Heath + +Coordinator +Vega 2.005 +972-883-7453 +jennie.heath@utdallas.edu + +Business Services +Matthew Grief + +Associate Vice President for Student Affairs +SSA 14.120G +972-883-6232 +grief@utdallas.edu + +Stan Scroggins + +Senior Director of Business Services +Helix 2.005 +972-883-7313 +stan.scroggins@utdallas.edu + +Ryan Hopkins + +Business Administrator +Helix 2.004 +972-883-4801 +ryan.hopkins@utdallas.edu + +Ericka Butler + +Financial Operations Coordinator +Helix 2.006 +972-883-6352 +ericka.butler@utdallas.edu + +Colen Skinner + +Financial Operations Coordinator +972-883-7309 +colen.skinner@utdallas.edu + +Todd Lawrence + +Financial Operations Coordinator +Helix 2.007 +972-883-7325 +todd.lawrence@utdallas.edu + +Assessment and Guest Housing +Matthew Grief + +Associate Vice President for Student Affairs +SSA 14.120G +972-883-6232 +grief@utdallas.edu + +Ryan White + +Assistant Vice President of Residential Life +Andromeda 1.405 +972-883-7430 +ryan.white@utdallas.edu + +Elizabeth Johnson + +Associate Director of Assessment and Guest Housing +Andromeda 1.402 +972-883-7427 +elizabeth.johnson@utdallas.edu + +Ren Lafleur + +Administrative Project Coordinator I +Vega 1.006 +972-883-7421 +ren.lafleur@utdallas.edu \ No newline at end of file diff --git a/data/housing/housing_staff_leadership.txt b/data/housing/housing_staff_leadership.txt new file mode 100644 index 0000000..f49a86b --- /dev/null +++ b/data/housing/housing_staff_leadership.txt @@ -0,0 +1,420 @@ +URL: https://housing.utdallas.edu/housing-staff-leadership +Title: Housing Leadership | UTD +Depth: 1 + +------------------------------------------------------------ + +Housing Leadership +Housing Offices +Housing Leadership +Staff Listings + +The University Housing leadership staff provides oversight for all campus housing. UT Dallas departments should contact a member of the University Housing team listed below. Residents who feel their needs have not been addressed by University Housing staff can also select from this listing. + +Mission Statement + +University Housing strives to provide an exceptional residential experience by delivering educational opportunities and supportive services that foster diverse engagement, individual growth and community belonging. + +Leadership +Matthew Grief + +Associate Vice President for Student Affairs +SSA 14.120G +972-883-6232 +grief@utdallas.edu + +Ryan White + +Assistant Vice President of Residential Life +Andromeda 1.405 +972-883-7430 +ryan.white@utdallas.edu + +Hayley Minton + +Senior Director of Housing Operations +UVHO 1.830A +972-883-7300 +hminton@utdallas.edu + +Stan Scroggins + +Senior Director of Business Services +Helix 2.005 +972-883-7313 +stan.scroggins@utdallas.edu + +Kaytie Harwood + +Director of Residential Life +Andromeda 1.407 +972-883-7430 +kaytie.harwood@utdallas.edu + +Sarah Mirarchi + +Associate Director of Housing Administration +Andromeda 1.414 +972-883-7412 +sarah.mirarchi@utdallas.edu + +Taylor Lawson + +Associate Director of Housing Operations +UVHO 1.875 +972-883-7300 +taylor.lawson@utdallas.edu + +Andres DeJesus + +Associate Director of Housing Operations +HSW 1.225 +972-883-7300 +andres.dejesus@utdallas.edu + +Elizabeth Johnson + +Associate Director of Assessment and Guest Housing +Andromeda 1.402 +972-883-7427 +elizabeth.johnson@utdallas.edu + +Seth Harwood + +Program Manager for Risk Management +UVHO 1.853 +972-883-7300 +seth.harwood@utdallas.edu + +Christina Schwiderski + +Associate Director of Residential Education +Andromeda 1.415 +972-883-7430 +christina.schwiderski@utdallas.edu + +Residential Life +Matthew Grief + +Associate Vice President for Student Affairs +SSA 14.120G +972-883-6232 +grief@utdallas.edu + +Ryan White + +Assistant Vice President of Residential Life +Andromeda 1.405 +972-883-7430 +ryan.white@utdallas.edu + +Kaytie Harwood + +Director of Residential Life +Andromeda 1.407 +972-883-7430 +kaytie.harwood@utdallas.edu + +Christina Schwiderski + +Associate Director of Residential Education +Andromeda 1.415 +972-883-7430 +christina.schwiderski@utdallas.edu + +Lisa Koontz + +Assistant Director of Apartment Area +Andromeda 1.413 +972-883-7430 +lisa.koontz@utdallas.edu + +Luis Robles + +Assistant Director of University Commons +Andromeda 1.411 +972-883-7430 +luis.robles@utdallas.edu + +Maeve Benson + +Residential Life Coordinator +Capella Hall + +Capella 1.006 +972-883-7429 +mbenson@utdallas.edu + +ME Lasso de la Vega + +Residential Life Coordinator +Andromeda Hall +Andromeda 1.003 +972-883-7424 +maria.emilia@utdallas.edu + +Victoria Nord + +Residential Life Coordinator +Buildings 2-31 & Canyon Creek Heights +CCHS 1.118 + +972-883-7332 +victoria.nord@utdallas.edu + +Alexis Rodriguez + +Residential Life Coordinator +Buildings 33-51 +Phase 8 Office +972-883-7423 +alexis.rodriguez3@utdallas.edu + +Ren Lafleur + +Assistant Residential Life Coordinator +Vega Hall +Vega 1.006 +972-883-7421 +ren.lafleur@utdallas.edu + +Housing Operations +Matthew Grief + +Associate Vice President for Student Affairs +SSA 14.120G +972-883-6232 +grief@utdallas.edu + +Hayley Minton + +Senior Director of Housing Operations +UVHO 1.830A +972-883-7300 +hminton@utdallas.edu + +Taylor Lawson + +Associate Director of Housing Operations +UVHO 1.875 +972-883-7300 +taylor.lawson@utdallas.edu + +Andres DeJesus + +Associate Director of Housing Operations +HSW 1.225 +972-883-7300 +andres.dejesus@utdallas.edu + +Seth Harwood + +Program Manager for Risk Management +UVHO 1.853 +972-883-7300 +seth.harwood@utdallas.edu + +Ashleigh Beckmann + +Assistant Director of Housing Operations +UVHO 1.860 +972-883-7306 +ashleigh.beckmann@utdallas.edu + +Jackson Havis + +Assistant Superintendent +of Housing Operations +UC Maintenance +972-883-7300 +jackson.havis@utdallas.edu + +Michael Crocker + +Assistant Superintendent of Housing Operations +CCHS 1.114 +972-883-7300 +michael.crocker@utdallas.edu + +Cesar Arreola + +Assistant Superintendent +of Housing Operations +UVHO 1.805 +972-883-7300 +cesar.arreola@utdallas.edu + +Maria Mares + +Building Services Supervisor +HSW 1.210 +972-883-7300 +maria.mares@utdallas.edu + +Leonard Griffin + +Inventory Control Specialist +HSW 1.130 +972-883-7300 +lxg132230@utdallas.edu + +Housing Administration +Matthew Grief + +Associate Vice President for Student Affairs +SSA 14.120G +972-883-6232 +grief@utdallas.edu + +Ryan White + +Assistant Vice President of Residential Life +Andromeda 1.405 +972-883-7430 +ryan.white@utdallas.edu + +Sarah Mirarchi + +Associate Director of Housing Administration +Andromeda 1.414 +972-883-7412 +sarah.mirarchi@utdallas.edu + +Emily Ross + +Coordinator of Housing Administration +Andromeda 1.408 +972-883-7413 +emily.ross@utdallas.edu + +Emily Rigsby + +Coordinator of Housing Administration +Andromeda 1.410 +972-883-7430 +emily.rigsby@utdallas.edu + +Vacant + +Coordinator of Housing Administration +Andromeda 1.408 + +Living Learning Communities +Matthew Grief + +Associate Vice President for Student Affairs +SSA 14.120G +972-883-6232 +grief@utdallas.edu + +Ryan White + +Assistant Vice President of Residential Life +Andromeda 1.405 +972-883-7430 +ryan.white@utdallas.edu + +Christina Schwiderski + +Associate Director of Residential Education +Andromeda 1.415 +972-883-7430 +christina.schwiderski@utdallas.edu + +Mariessa Joseph + +Assistant Director of Living Learning Communities +Andromeda +972-883-7430 +mariessa.joseph@utdallas.edu + +Madison Polk + +Coordinator +Vega 2.004 +madison.polk@utdallas.edu + +Nicky Dang + +Coordinator +Vega 2.007 +972-883-7456 +nicky.dang@utdallas.edu + +Jennie Heath + +Coordinator +Vega 2.005 +972-883-7453 +jennie.heath@utdallas.edu + +Business Services +Matthew Grief + +Associate Vice President for Student Affairs +SSA 14.120G +972-883-6232 +grief@utdallas.edu + +Stan Scroggins + +Senior Director of Business Services +Helix 2.005 +972-883-7313 +stan.scroggins@utdallas.edu + +Ryan Hopkins + +Business Administrator +Helix 2.004 +972-883-4801 +ryan.hopkins@utdallas.edu + +Ericka Butler + +Financial Operations Coordinator +Helix 2.006 +972-883-6352 +ericka.butler@utdallas.edu + +Colen Skinner + +Financial Operations Coordinator +972-883-7309 +colen.skinner@utdallas.edu + +Todd Lawrence + +Financial Operations Coordinator +Helix 2.007 +972-883-7325 +todd.lawrence@utdallas.edu + +Assessment and Guest Housing +Matthew Grief + +Associate Vice President for Student Affairs +SSA 14.120G +972-883-6232 +grief@utdallas.edu + +Ryan White + +Assistant Vice President of Residential Life +Andromeda 1.405 +972-883-7430 +ryan.white@utdallas.edu + +Elizabeth Johnson + +Associate Director of Assessment and Guest Housing +Andromeda 1.402 +972-883-7427 +elizabeth.johnson@utdallas.edu + +Ren Lafleur + +Administrative Project Coordinator I +Vega 1.006 +972-883-7421 +ren.lafleur@utdallas.edu \ No newline at end of file diff --git a/data/housing/index.json b/data/housing/index.json new file mode 100644 index 0000000..54cef8f --- /dev/null +++ b/data/housing/index.json @@ -0,0 +1,181 @@ +{ + "source": "https://housing.utdallas.edu/", + "total_pages": 35, + "pages": [ + { + "url": "https://housing.utdallas.edu/", + "title": "University Housing Home | UTD", + "depth": 0 + }, + { + "url": "https://housing.utdallas.edu/housing-staff", + "title": "Housing Staff | UTD", + "depth": 1 + }, + { + "url": "https://housing.utdallas.edu/housing-departments/guest-housing", + "title": "Corporate Intern and Summer Guest Housing | UTD", + "depth": 1 + }, + { + "url": "https://housing.utdallas.edu/1877/university-housing-updates", + "title": "University Housing Updates - University Housing | The University of Texas at Dallas", + "depth": 1 + }, + { + "url": "https://housing.utdallas.edu/housing-departments", + "title": "Housing Departments | UTD", + "depth": 1 + }, + { + "url": "https://housing.utdallas.edu/housing-options", + "title": "Housing Options | UTD", + "depth": 1 + }, + { + "url": "https://housing.utdallas.edu/resources/first-year-housing", + "title": "First-Year Housing | UTD", + "depth": 1 + }, + { + "url": "https://housing.utdallas.edu/residential-technology", + "title": "Residential Technology | University Housing | UTD", + "depth": 1 + }, + { + "url": "https://housing.utdallas.edu/returning-resident-application", + "title": "Returning Resident Application | UTD", + "depth": 1 + }, + { + "url": "https://housing.utdallas.edu/resources/resident-handbook", + "title": "Resident Handbook | UTD", + "depth": 1 + }, + { + "url": "https://housing.utdallas.edu/housing-options/canyon-creek-heights", + "title": "Canyon Creek Heights | Housing | UTD", + "depth": 1 + }, + { + "url": "https://housing.utdallas.edu/resources/move-in", + "title": "Move-In", + "depth": 1 + }, + { + "url": "https://housing.utdallas.edu/housing-options/housing-availability", + "title": "Housing Availability | UTD", + "depth": 1 + }, + { + "url": "https://housing.utdallas.edu/resources", + "title": "Resources | Housing | UTD", + "depth": 1 + }, + { + "url": "https://housing.utdallas.edu/contact", + "title": "Housing Offices | UTD", + "depth": 1 + }, + { + "url": "https://housing.utdallas.edu/housing-options/university-commons", + "title": "University Commons | Housing | UTD", + "depth": 1 + }, + { + "url": "https://housing.utdallas.edu/housing-staff-leadership", + "title": "Housing Leadership | UTD", + "depth": 1 + }, + { + "url": "https://housing.utdallas.edu/housing-options/university-village", + "title": "University Village | Housing | UTD", + "depth": 1 + }, + { + "url": "https://housing.utdallas.edu/operations", + "title": "Operations | Housing | UTD", + "depth": 1 + }, + { + "url": "https://housing.utdallas.edu/housing-departments/guest-housing/what-to-bring-summer-guest-housing", + "title": "What To Bring: Summer Guest Housing | UTD", + "depth": 2 + }, + { + "url": "https://housing.utdallas.edu/housing-departments/guest-housing/amenities-guest-housing", + "title": "Amenities: Guest Housing | UTD", + "depth": 2 + }, + { + "url": "https://housing.utdallas.edu/housing-departments/guest-housing/housing-options-guest", + "title": "Housing Options: Interns & Summer Guests - University Housing | The University of Texas at Dallas", + "depth": 2 + }, + { + "url": "https://housing.utdallas.edu/housing-departments/guest-housing/policies-summer-guest-housing", + "title": "Policies: Summer Guest Housing - University Housing | The University of Texas at Dallas", + "depth": 2 + }, + { + "url": "https://housing.utdallas.edu/housing-departments/guest-housing/pricing-summer-guest-housing", + "title": "Pricing: Summer Guest Housing - University Housing | The University of Texas at Dallas", + "depth": 2 + }, + { + "url": "https://housing.utdallas.edu/housing-departments/guest-housing/guest-housing-resources", + "title": "Guest Housing Resources | UTD", + "depth": 2 + }, + { + "url": "https://housing.utdallas.edu/category/uncategorized", + "title": "Uncategorized - University Housing | The University of Texas at Dallas", + "depth": 2 + }, + { + "url": "https://housing.utdallas.edu/resources/move-in/what-to-bring", + "title": "Move-in: Checklist of items | Housing | UTD", + "depth": 2 + }, + { + "url": "https://housing.utdallas.edu/canyon-creek-heights-pricing-and-floorplans", + "title": "Canyon Creek Heights Pricing and Floor Plans | University Housing | UTD", + "depth": 2 + }, + { + "url": "https://housing.utdallas.edu/resources/move-in/before-you-arrive", + "title": "Move-In: Before You Arrive | Housing | UT Dallas", + "depth": 2 + }, + { + "url": "https://housing.utdallas.edu/resources/move-in/instructions-and-tips", + "title": "Move-In: Instructions & Tips | Housing | UT Dallas", + "depth": 2 + }, + { + "url": "https://housing.utdallas.edu/resources/move-in/floorplans-and-dimensions", + "title": "Move-In: Floorplans & Dimensions | Housing | UTD", + "depth": 2 + }, + { + "url": "https://housing.utdallas.edu/resources/move-in/lodging-and-restaurants", + "title": "Move-In: Local Lodging & Restaurants | University Housing - UTD", + "depth": 2 + }, + { + "url": "https://housing.utdallas.edu/university-village-pricing-and-floorplans", + "title": "University Village Pricing and Floor Plans | Housing | UTD", + "depth": 2 + }, + { + "url": "https://housing.utdallas.edu/housing-departments/corporate-intern-and-summer-guest-housing", + "title": "Corporate Intern and Summer Guest Housing | UTD", + "depth": 2 + }, + { + "url": "https://housing.utdallas.edu/university-commons-pricing-and-floorplans", + "title": "University Commons Pricing and Floor Plan | Housing | UTD", + "depth": 2 + } + ] +} \ No newline at end of file diff --git a/data/housing/index.txt b/data/housing/index.txt new file mode 100644 index 0000000..31680c3 --- /dev/null +++ b/data/housing/index.txt @@ -0,0 +1,93 @@ +URL: https://housing.utdallas.edu/ +Title: University Housing Home | UTD +Depth: 0 + +------------------------------------------------------------ + +Canyon Creek Heights +Andromeda Hall (RHA) +University Village +Canyon Creek Heights +Andromeda Hall (RHA) +1 +2 +3 +Previous +Next +First-Year Priority Application Deadline + +The priority application deadline for first-year students is April 15, 2026. All completed applications received prior to that deadline receive priority based on the date of application. After April 15, consideration is given based on the date of application and proximity to campus. + +Non-First Year 2025-2026 Housing Availability + +View the current housing availability for the 2025-2026 academic year. + +Housing Agreement Changes & Dates + +View the 2025-2026 University Housing Terms and Conditions. (PDF) + +The following dates are considered the Term of your Agreement if your University Housing Terms and Conditions is received and accepted by University Housing and you are provided with a housing assignment: + +Summer Only Agreement: May 18, 2025 – July 31, 2025 or 48 hours after written cancellation notice is provided, whichever comes first +Academic Year Agreement: Aug. 25, 2025 – May 16, 2026 +Mid-Year Spring Only Agreement: Jan. 20, 2026 – May 16, 2026 + +View the 2026-2027 University Housing Terms and Conditions. (PDF) +View the 2026 Summer Addendum. (PDF) + +The following dates are considered the Term of your Agreement if your University Housing Terms and Conditions is received and accepted by University Housing and you are provided with a housing assignment: + +Summer Only Agreement: May 17, 2026 – July 31, 2026 or 48 hours after written cancellation notice is provided, whichever comes first +Academic Year Agreement: Aug. 24, 2026 – May 15, 2027 +Mid-Year Spring Only Agreement: Jan. 19, 2027 – May 15, 2027 + +University Housing strives to provide an exceptional residential experience by delivering educational opportunities and supportive services that foster diverse engagement, individual growth and community belonging. + +APPLY NOW! +Important Updates + +12/18/2025 +View our Winter Break hours here. + +10/7/2025 +Please make note of the following housing application deadlines. Applications will be available in the MyHousing portal. + +Oct. 1, 2025: Spring 2026 Application opens. (New residents only. For Spring 2026 housing only.) +Nov. 3, 2025:  First-Year Housing Application opens. (First-year students only. For Fall 2026 – Spring 2027.)  +Jan 20, 2026: Sign up to be notified when the New Resident Housing Application opens for Fall 2026 – Spring 2027. (New residents only. For Fall 2026 – Spring 2027.)  +Jan. 20 – Mar. 6, 2026 (Planned dates, subject to change):  Renewal and Sign-Up. (Returning residents only. For Fall 2026 – Spring 2027.) +Jan. 20-23, 2026: Renewal Application available. +Feb. 2-6, 2026: Intent to Sign Up Application available. +Feb. 16-20, 2026: Roommate Groups Application available. +Mar. 2-6, 2026: Room Selection Application available.  +Apr. 15, 2026: First-Year Housing Application priority deadline. (First-year students only. For the Fall 2026 – Spring 2027 academic year.)  + +7/31/2025 +View University Housing’s updated Electric Recreational Mobility Device Policy. + +View additional updates + +University Commons + +Residence Halls + +University Commons residence halls feature three-bedroom suites, recreation areas, large communal spaces and study lounges. Available to first-year students*. + +University Village + +Unfurnished Apartments + +University Village apartments offer a variety of floor plans and amenities. Available to non-first year students. + +Canyon Creek Heights + +Furnished Apartments + +Canyon Creek Heights apartments offer a variety of floor plans and amenities. Available to non-first year students. + +* First-year students are those who graduate high school in May or June prior to their first semester at UT Dallas. + +First-Year Housing +Family Housing +Summer Guest Housing +Off-Campus Housing \ No newline at end of file diff --git a/data/housing/operations.txt b/data/housing/operations.txt new file mode 100644 index 0000000..a716d24 --- /dev/null +++ b/data/housing/operations.txt @@ -0,0 +1,29 @@ +URL: https://housing.utdallas.edu/operations +Title: Operations | Housing | UTD +Depth: 1 + +------------------------------------------------------------ + +Operations + +The office of the associate vice president for Student Affairs oversees all housing operations on campus, including all residence halls and apartments. Operations’ main goals include: + +Providing quality oversight of all housing operations. +Improving housing conditions on campus through intensive, long-range capital planning. +Delivering quality facilities for UT Dallas students to eat, live and study. + +Maintenance & Housekeeping + +Project Updates + +Residential Technology + +University Housing strives to provide an exceptional residential experience by delivering educational opportunities and supportive services that foster diverse engagement, individual growth and community belonging. + +Work Orders + +Routine Work Orders +Regular, non-emergency work orders can be submitted through the MyHousing Portal. Requests will be addressed within 24-48 hours. + +Emergency Work Orders +Emergencies are issues that threaten life, health, or property (ex: water leak, overflowing toilet, smoke, fire, etc.). Emergency work orders must be placed in person or called into the appropriate building. On-call staff are available to answer after-hours emergencies. \ No newline at end of file diff --git a/data/housing/residential_technology.txt b/data/housing/residential_technology.txt new file mode 100644 index 0000000..d4ad5b4 --- /dev/null +++ b/data/housing/residential_technology.txt @@ -0,0 +1,57 @@ +URL: https://housing.utdallas.edu/residential-technology +Title: Residential Technology | University Housing | UTD +Depth: 1 + +------------------------------------------------------------ + +Residential Technology + +University Housing provides residents with wireless and wired internet through Boldyn. + +Get Connected + +Step 1: Download Duo Mobile + +Download the Duo Mobile Authentication App on your phone. + +Step 2: Access the Online Portal + +If connecting for the first time, open your wireless settings and connect to “MyResNet Start Here”. +If reconnecting, go to Boldyn’s online portal. + +Step 3: Sign In + +Sign into the portal using your UT Dallas email address and password. +Accept the Duo notification push. +Accept Boldyn’s terms and complete your profile. +Note the Wi-Fi passkey that pops up on your screen. +To change your password, click “show Wi-Fi password” under the “Add Devices” tab. This is a private password and should not be shared. + +Step 4: Connect to the Network + +Open your wireless settings on each device you are connecting. +Connect to MyResNet-5G. (Only connect to MyResNet-2G if you cannot see the 5G network.) +Enter your Wi-Fi password when prompted. +Enable the “auto-join” feature. + +Step 5: Connecting Other Devices + +For a wired device, or a device without a graphical user interface (GUI), login to the portal and manually add the MAC address in the My Devices tab. +Internet Support + +Boldyn provides 24/7 customer support for all of your internet needs. + +Call 855-465-6750. +Live Chat dallasoncampus.hed.boldyn.com. +Email support.hed@boldyn.com for additional assistance. +Guest Network + +Guests and visitors can connect to the guest network for up to 12 hours. + +Step 1: From your device, click on “MyResNet-Start-Here” in your available networks. If the network is not available, go to dallasoncampus.hed.boldyn.com + +Step 2: In the new portal, click “Guests and Visitors”. + +Step 3: Click “Continue”. + +Step 4: Check the box to agree to the terms, then click “Create Guest Account”. \ No newline at end of file diff --git a/data/housing/resources.txt b/data/housing/resources.txt new file mode 100644 index 0000000..7fee928 --- /dev/null +++ b/data/housing/resources.txt @@ -0,0 +1,44 @@ +URL: https://housing.utdallas.edu/resources +Title: Resources | Housing | UTD +Depth: 1 + +------------------------------------------------------------ + +Resources +MyHousing Portal + +The MyHousing portal is University Housing’s one-stop-shop for everything regarding on-campus housing. Students may use the portal to: + +Apply/sign-up for on-campus housing +Participate in the Returning Resident application process +Check the status of their housing application +Submit work orders +First-Year Housing + +First-year students who wish to live on campus are assigned to one of the University Commons residence halls. First-year students are defined as those who graduate high school in May or June prior to their first semester at UT Dallas. + +Family Housing + +Limited family housing is available through University Housing for UT Dallas students living with spouses and/or children. These units are offered on a first-come, first-served basis, and availability cannot be guaranteed. + +If you’re interested in family housing, contact the University Housing Office to inquire about current availability. + +Please note: + +All individuals residing in a family housing unit must sign an apartment housing agreement. +Canyon Creek Heights and University Commons are single bedroom housing options and are not available as family housing. +Corporate Intern & Summer Guest Housing + +Corporate Intern & Summer Guest Housing offers housing to corporate interns and non-UTD student summer researchers. + +Off-Campus Housing + +The Dallas Off-Campus Housing database is provided as a service to The University of Texas at Dallas community and is for informational purposes only. + +Resident Handbook + +The Resident Handbook is a reference source for University Housing that discusses the safety and well-being of the community. + +Residential Technology + +University Housing provides residents with wireless and wired internet through Boldyn. Visit the Residential Technology website for information on getting connected and contacting Boldyn’s 24/7 customer support. \ No newline at end of file diff --git a/data/housing/resources_first_year_housing.txt b/data/housing/resources_first_year_housing.txt new file mode 100644 index 0000000..b326e1d --- /dev/null +++ b/data/housing/resources_first_year_housing.txt @@ -0,0 +1,62 @@ +URL: https://housing.utdallas.edu/resources/first-year-housing +Title: First-Year Housing | UTD +Depth: 1 + +------------------------------------------------------------ + +First-Year Housing + +First-year students who want to live on campus are assigned to one of the University Commons residence halls. First-year students are defined as those who graduate high school in May or June prior to their first semester at UT Dallas. + +Application Process + +Applicants who do not submit a completed application will not be eligible to receive a housing assignment. On-campus housing for first-year students is not guaranteed. UT Dallas does not have a live-on requirement for first-year students. + +The priority application deadline is April 15, 2026. All completed applications received prior to that deadline receive priority based on the date of application. After April 15, consideration is given based on the date of application and proximity to campus. + +Students must be admitted to the University before they can apply for housing. Visit the Enrollment Services website to apply for admission to UT Dallas. +Complete the housing application online. Please note the $100 administrative fee is non-refundable. +Students who will not be 18 years old prior to move-in will need a parent or legal guardian to sign the housing agreement on their behalf. +A complete application consists of the completed application, the signed housing agreement, and the $100 administrative fee payment. +Click here for an overview of the First-Year Application. (Stay tuned for part 3 coming in late Spring 2026!) +After Submitting Your Application + +Applications will be given priority based on availability and according to the time and date that the application is considered complete. A complete application consists of the completed application, the signed housing agreement, and the $100 administrative fee payment. For students who will not be 18 years old prior to move-in, a parent or legal guardian will need to sign the housing agreement on their behalf. + +Housing Assignments + +Suite assignments for the fall semester will be sent to your UT Dallas email address in July. Names and emails of roommates will be made available in the MyHousing portal. Assignments and roommate information will not be released over the phone. + +First-year students who are admitted for summer or spring will be provided with move-in information from University Housing staff. + +Roommate Assignments +Roommates are assigned based on gender and space availability. All buildings are coed; individual suites are not. +Applicants will be given priority based on availability and according to the date and time the completed application and fees are submitted (prior to April 15). +Every effort is made to place students who mutually request each other as roommates into the same suite. There are many factors that determine whether a roommate group will be placed together including, but not limited to, date of application, Honors Housing, or LLC Housing. +While roommate groups are not guaranteed, we do our best to honor as many requests as possible. +If no roommate is requested, students will be matched based on answers to questions on their housing applications and the date of their completed application. +Assignments are made without regard to race, color, religion, national origin, disability, sexual orientation or veteran status. +Living Learning Community selection overrides roommate requests and Honors Housing. +University Housing reserves the right to make housing assignment changes as necessary. +First-Year Housing Policies +Cancellation Policy + +Residents who cancel their housing agreement are subject to the Cancellation Policy found in their University Housing Terms and Conditions. Read the Cancellation Policy. + +All cancellations must be submitted through the Cancellation form available in the MyHousing portal. +In all cases, the $100 administrative fee is nonrefundable. +Residence Hall Policies + +Refer to the Resident Handbook and the Student Code of Conduct page for policy information. + +Occupancy + +The maximum occupancy for three-bedroom suites is three occupants. + +Housing Payment + +Residents will have their housing charge placed on their student account through the Bursars Office. For information regarding payment methods and deadlines, visit the Bursars Office website. + +Insurance + +It is recommended that students obtain personal property, renters, or other applicable insurance. The University of Texas at Dallas does not assume any legal obligations for personal injury, loss, or damage to personal property. \ No newline at end of file diff --git a/data/housing/resources_move_in.txt b/data/housing/resources_move_in.txt new file mode 100644 index 0000000..253f02f --- /dev/null +++ b/data/housing/resources_move_in.txt @@ -0,0 +1,49 @@ +URL: https://housing.utdallas.edu/resources/move-in +Title: Move-In +Depth: 1 + +------------------------------------------------------------ + +Move-In Resources + +Fall 2025 Move-In Dates + +Monday 8/18: Andromeda Hall, Sirius Hall, Capella Hall +Tuesday 8/19: Andromeda Hall, Sirius Hall, Capella Hall +Wednesday 8/20: Helix Hall, Vega Hall, Canyon Creek Heights North +Thursday 8/21: Helix Hall, Vega Hall, Canyon Creek Heights South +Saturday 8/23: University Village + +Housing assignments will be sent to the students’ UT Dallas email accounts the first week in July. Emails to first-year University Commons residents will include a link to sign up for their preferred move-in date and time. University Village and Canyon Creek Heights residents will receive a pre-assigned move-in date and time. + +Before You Arrive + +Everything you need to know +before you arrive on Move-In Day. + +Move-In Day Instructions & Tips + +Here’s how to have a stress-free move in day. + +What to Bring (And What NOT to Bring) + +Before you start packing, make sure you know which items are and are not allowed. + +Floorplans & Dimensions + +Get a feel for your space before you arrive. + +Local Lodging & Restaurants + +Need a place to stay before move-in? +Looking for a great place to eat? +We’ve got you. + +Resources & FAQs + +Check here for links to downloadable items and answers to our most common move-in questions. + +Don’t forget to follow us on social media! +Instagram +Facebook +YouTube \ No newline at end of file diff --git a/data/housing/resources_move_in_before_you_arrive.txt b/data/housing/resources_move_in_before_you_arrive.txt new file mode 100644 index 0000000..550df5b --- /dev/null +++ b/data/housing/resources_move_in_before_you_arrive.txt @@ -0,0 +1,51 @@ +URL: https://housing.utdallas.edu/resources/move-in/before-you-arrive +Title: Move-In: Before You Arrive | Housing | UT Dallas +Depth: 2 + +------------------------------------------------------------ + +Before You Arrive +Resources > Move-In > Before You Arrive + +Review your housing assignment email + +All residents will receive a housing assignment email in their UT Dallas email. Make sure to read through the email, as it contains important information about: + +Housing assignment and mailing address +Move-in date (University Commons residents will receive a link to sign up for their preferred move-in date and time) +Roommate information +Renters insurance +Meal plan +Mini fridge, microwave, and linen rentals +Housing payments +Resident Handbook +Housing eligibility +Cancellation + +Contact your roommates + +Your housing assignment email has your roommates’ contact information. Touch base with them before you arrive to get to know each other and discuss who will bring what. Does your roommate have a mini fridge that they’d like to bring? Do you want to bring a table for the living area? + +Know what TO bring and what NOT to bring + +We’ve got a whole webpage with this information. Print out the pdf and take it when you go shopping! Speaking of shopping, you’re probably going to need less than you think. Our first-year residence halls (University Commons) are already fully furnished, so no additional furniture is needed. We always suggest waiting until you move in and see your space before buying big items and decorations. Target and Walmart are both within a 5-minute drive from campus. + +Please note that items may not be shipped to your on-campus address before you move in. Additionally, University Housing does not accept furniture shipments due to available storage space. Residents ordering furniture must make arrangements to meet the delivery driver outside. + +Label your belongings + +Make sure to label your items in case anything gets lost. You can print labels and other helpful documents from our Resources & FAQs page. + +Don’t forget your Comet Card! + +Your Comet Card is your key to your building and your suite. You also need it to move in! So don’t pack it away in a box. Keep it in a place that is easily accessible so you can grab it when you arrive on campus. + +Resident Handbook + +Familiarize yourself with the Resident Handbook. All residents agree to follow the policies and procedures set forth in the Resident Handbook. + +Be Weather Aware + +While we cannot predict the weather with absolute certainty, we do know that it will be HOT in August! Wear loose and light clothing, drink plenty of water, and bring sunscreen. Stay in the shade or indoors as much as possible, and take frequent breaks to avoid overheating. + +Please be aware that if thunderstorms are in the area, move-in assistance will be postponed until the lightening has passed. \ No newline at end of file diff --git a/data/housing/resources_move_in_floorplans_and_dimensions.txt b/data/housing/resources_move_in_floorplans_and_dimensions.txt new file mode 100644 index 0000000..5dd9aed --- /dev/null +++ b/data/housing/resources_move_in_floorplans_and_dimensions.txt @@ -0,0 +1,107 @@ +URL: https://housing.utdallas.edu/resources/move-in/floorplans-and-dimensions +Title: Move-In: Floorplans & Dimensions | Housing | UTD +Depth: 2 + +------------------------------------------------------------ + +Floorplans & Dimensions +Resources > Move-In > Floorplans & Dimensions + +Floorplans + +All measurements are approximate and may vary slightly from one unit to another. If you are unsure whether an item will fit, we suggest waiting until you move in and take measurements yourself. Please note that we are unable to show incoming residents their rooms prior to their assigned move-in day due to our maintenance and housekeeping operations. Before you go shopping, remember to check out our “What to Bring/Not to Bring” page to see what is allowed. + +UNIVERSITY COMMONS FLOORPLAN +UNIVERSITY VILLAGE FLOORPLANS +CANYON CREEK HEIGHTS FLOORPLANS + +Dimensions + +University Commons + +Furnished residence halls + +All measurements are approximate + +Bed Frame +Sized for Twin XL. Loftable. + +Mattress +Twin XL (80″L x 38″W) + +Bedroom Window +50.75″W x 61″H + +Desk +32″W x 30.25″H x 24.25″D + +Drawers +29.5″W x 30.25″H x 24.25″D + +Nightstand +17.25″W x 30.25″H x 24.25″D + +University +Village + +Unfurnished apartments + +All measurements are approximate + +University Village apartments are unfurnished. + +However, 4×2 units in Phases 8, 8A, and 9 have built-in desks that measure approximately 57.5″W x 23.5″D + +Canyon Creek Heights North + +Furnished apartments + +All measurements are approximate + +Bed Frame +Sized for Twin XL. Loftable. + +Mattress +Twin XL (80″L x 38″W) + +Bedroom Window +46″W x 73.5″H + +Desk +49.5″W x 24.25″D x 30″H + +Drawers +29.5″W x 30″H x 24.5″D + +Nightstand +17″W x 30″H x 24.5″D + +Common Area Coffee Table +32″W x 20″D x 28.75″H + +Canyon Creek Heights South + +Furnished apartments + +All measurements are approximate + +Bed Frame +Sized for Full. Loftable. + +Mattress +Full (75″L x 54″W) + +Bedroom Window +46″W x 73.5″H + +Desk +49.5″W x 24.25″D x 30″H + +Drawers +29.5″W x 30H x 24.5″D + +Nightstand +17″ W x 30″H x 24.5″D + +Common Area Coffee Table +32″W x 20″D x 28.75″H \ No newline at end of file diff --git a/data/housing/resources_move_in_instructions_and_tips.txt b/data/housing/resources_move_in_instructions_and_tips.txt new file mode 100644 index 0000000..136283b --- /dev/null +++ b/data/housing/resources_move_in_instructions_and_tips.txt @@ -0,0 +1,57 @@ +URL: https://housing.utdallas.edu/resources/move-in/instructions-and-tips +Title: Move-In: Instructions & Tips | Housing | UT Dallas +Depth: 2 + +------------------------------------------------------------ + +Move-In Day Instructions & Tips +Resources > Move-In > Instructions & Tips +Instructions + +Prepare + +Make sure your Comet Card is ready and accessible. +Label all of your boxes. +Check the weather forecast for the day. Be prepared for hot weather. In the event of thunderstorms, move-in assistance will be postponed until lightning has passed. + +Check-In + +Arrive on time to key pickup at the Activity Center. Arriving at your scheduled time helps prevent long lines and ensures our move-in operations run smoothly. Residents who arrive over 20 minutes late for their move-in time may need to wait until the end of the day to pick up their keys and complete their move-in. +University Commons residents will be able to select their preferred move-in date and time. Check the housing assignment email for the link. +Canyon Creek Heights and University Village residents will have received a pre-assigned move-in date and time. +Jump in the line for your building to check-in and receive your keys. Have your Comet Card ready to show staff. +Receive your keys, route cards, campus map, and other helpful information. Place the route cards on the dash of your vehicles. Residents will be given one route card per car, for a maximum of two cars. + +Move-In + +First-year students in University Commons or Canyon Creek Heights: +After picking up your keys at the Activity Center, follow the signs to the move-in holding zone and get in the line for your building. Once you are at the front of the line, follow the golf cart to your building. +Pull up to your building’s unloading zone and unload your vehicle. Two vehicles are allowed per resident, and repeat trips through the unloading zone are not allowed. Due to tight spaces, U-Hauls and trailers are not allowed through the unloading zone and must unload from Parking Lot T. If you unload from Parking Lot T, you will need to provide your own moving supplies (dolly, wagon, etc.). +You will have 15 minutes to unload your vehicle and take your items up to your suite. +At least one person must stay with the car to move it after unloading. After parking, feel free to meet up with the rest of your party. +After unloading your cart, place it in the hallway so our staff can pick it up and take it back to the unloading zone for other students to use. +Non first-year students in University Village or Canyon Creek Heights: +Residents may proceed directly to the unloading zones and/or parking lots near their building. + +Don’t Forget! + +Now it’s time to decorate! Have fun and make the space your own, but remember to follow the decorating rules in the Resident Handbook (see “Decorating/Painting/Wall Hangings” section, pgs. 13-14). +Did you bring a bike? Make sure to register it at your building’s front desk. You will need your bike’s serial number, make, model, and color(s). Registration is free but required. +Tips + +Arrive On Time + +Sorry, but early move-ins are not allowed, as our maintenance and housekeeping teams are busy preparing your room for your arrival. +If you cannot arrive on your chosen date, email us ahead of time to make alternate arrangements. +Make sure to arrive at the Activity Center at your assigned time. Residents who arrive over 20 minutes late for their move-in time may need to wait until the end of the day to pick up their keys and complete their move-in. + +Stay Cool & Hydrated + +Dallas gets HOT in August! Drink plenty of water, and wear loose, light clothing and sunscreen. +Stay in the shade or indoors as much as possible. Take frequent breaks to avoid overheating. + +Don’t Forget! + +Two things many people do not think to bring: basic tools & trash bags. +Keep in mind that you do not have to finish your move right away! Once you have your keys you can take as much time as you need to move items in and get set up. (Just remember that you can only go through our unloading zone once; you will need to unload items from additional trips yourself.) +Target and Walmart are within a 5-minute drive from campus if you need to get more supplies. \ No newline at end of file diff --git a/data/housing/resources_move_in_lodging_and_restaurants.txt b/data/housing/resources_move_in_lodging_and_restaurants.txt new file mode 100644 index 0000000..59e20dd --- /dev/null +++ b/data/housing/resources_move_in_lodging_and_restaurants.txt @@ -0,0 +1,31 @@ +URL: https://housing.utdallas.edu/resources/move-in/lodging-and-restaurants +Title: Move-In: Local Lodging & Restaurants | University Housing - UTD +Depth: 2 + +------------------------------------------------------------ + +Local Lodging & Restaurants +Resources > Move-In > Local Lodging & Restaurants +Lodging + +The Office of Admission and Enrollment has a fantastic list of area hotels, many of which offer a special UT Dallas rate: enroll.utdallas.edu/visit/area-hotels. + +Restaurants + +On-Campus Options + +From the Dining Hall to Panda Express and Halal Shack, there are tons of places to eat on campus! Check out dineoncampus.com/utdallasdining for a full list of open dining locations. + +Off-Campus Options + +Richardson has a wide variety of restaurants to satisfy your every craving. Here are some that our Housing staff recommends within a 5-minute drive from campus. Don’t forget to ask if they provide discounts for UT Dallas students! + +(We limited our recommendations to 10 restaurants per category, but there are so many more great places to eat! A quick Google Maps search for area restaurants will pull up a more extensive list.) + +* Denotes Housing staff favorites + +Fast Food +Casual Sit Down +Sit Down +Breakfast/Brunch +Coffee/Tea/Dessert \ No newline at end of file diff --git a/data/housing/resources_move_in_what_to_bring.txt b/data/housing/resources_move_in_what_to_bring.txt new file mode 100644 index 0000000..1cfe080 --- /dev/null +++ b/data/housing/resources_move_in_what_to_bring.txt @@ -0,0 +1,11 @@ +URL: https://housing.utdallas.edu/resources/move-in/what-to-bring +Title: Move-in: Checklist of items | Housing | UTD +Depth: 2 + +------------------------------------------------------------ + +What To Bring (And What NOT To Bring) +Resources > Move-In > What to Bring +Click to open/download + +Click to open/download \ No newline at end of file diff --git a/data/housing/resources_resident_handbook.txt b/data/housing/resources_resident_handbook.txt new file mode 100644 index 0000000..c6a7bc0 --- /dev/null +++ b/data/housing/resources_resident_handbook.txt @@ -0,0 +1,26 @@ +URL: https://housing.utdallas.edu/resources/resident-handbook +Title: Resident Handbook | UTD +Depth: 1 + +------------------------------------------------------------ + +Resident Handbook + +The Resident Handbook (pdf) is a reference source for University Housing that discusses the safety and well-being of the community. Topics covered include (but are not limited to): + +University Housing conduct code +Office hours and closings +Maintenance +Emergencies +Bicycles +Health and safety inspections +Cleanliness, health and hygiene +Emergency call stations (Code Blue) +Grills and outdoor cooking appliances +Noise +Missing Residential Student Policy + +The purpose of this policy is to establish procedures for the university’s response to reports of missing residential students as required by the Higher Education Opportunity Act of 2008. This policy applies to all students who live in University Housing. + +MISSING RESIDENTIAL STUDENT FORM (PDF) +DOWNLOAD (PDF) \ No newline at end of file diff --git a/data/housing/returning_resident_application.txt b/data/housing/returning_resident_application.txt new file mode 100644 index 0000000..e747cbd --- /dev/null +++ b/data/housing/returning_resident_application.txt @@ -0,0 +1,16 @@ +URL: https://housing.utdallas.edu/returning-resident-application +Title: Returning Resident Application | UTD +Depth: 1 + +------------------------------------------------------------ + +Returning Resident Application + +Current University Housing residents have the option to participate in one of two Returning Resident Application processes: Renewal or Sign Up. This Returning Resident Application process allows residents to apply for housing for the summer 2026 term and the 2026-2027 academic year. + +View the Renewal and Sign Up Application walk-throughs on our YouTube channel. + +default +Renewal +Sign Up +FAQs \ No newline at end of file diff --git a/data/housing/university_commons_pricing_and_floorplans.txt b/data/housing/university_commons_pricing_and_floorplans.txt new file mode 100644 index 0000000..34f2cde --- /dev/null +++ b/data/housing/university_commons_pricing_and_floorplans.txt @@ -0,0 +1,45 @@ +URL: https://housing.utdallas.edu/university-commons-pricing-and-floorplans +Title: University Commons Pricing and Floor Plan | Housing | UTD +Depth: 2 + +------------------------------------------------------------ + +University Commons Pricing and Floor Plan +Pricing +Floor Plan1 Bed/Bath Square Feet1 Price Per Bed, Per Semester2 +Three Bedroom Suite 3/1 655 $4,600 +University Commons Pricing and Floor Plan +1Size information of floor plan is average for typical apartment. Exact dimensions, square footage and layouts vary. +2The following rates are planned for on-campus University Housing options for the 2025-2026 academic year. Rates are subject to change. Price includes all utilities. +Floor plan +Three-bedroom Suite* + +Private bedroom — $4,600 per semester, all utilities included; academic year agreement, summer additional. + +*Suite is furnished. Measurements are approximate. Floors plans may vary. +RELATED LINKS + +Features & Amenities + +Download Floor Plan (pdf) + +Resident Handbook + +First-Year Freshmen + +Cancellation Policy (pdf) + +Housing Agreements + +All housing agreements are for the complete academic year. + +Students who live in University Commons will sign an academic year agreement (mid-August through mid-May), with an option to participate in summer housing if desired. + +All housing fees associated with assignments in these buildings will be processed through the Bursar Office. Housing fees for the full housing agreement period will be divided into two equal charges. Your University account will be charged once prior to the fall semester and once prior to the spring semester, in accordance with Bursar timelines. Summer charges for University Commons will be in addition to the listed rates. + +Students who move on campus prior to census day will be charged the full amount of housing for the semester. Charges will not be pro-rated until these dates: + +Spring 2026: Feb. 4 +Summer 2026: June 16 +Fall 2026: Sept. 9 +Spring 2027: Feb. 3 \ No newline at end of file diff --git a/data/housing/university_village_pricing_and_floorplans.txt b/data/housing/university_village_pricing_and_floorplans.txt new file mode 100644 index 0000000..45eec60 --- /dev/null +++ b/data/housing/university_village_pricing_and_floorplans.txt @@ -0,0 +1,56 @@ +URL: https://housing.utdallas.edu/university-village-pricing-and-floorplans +Title: University Village Pricing and Floor Plans | Housing | UTD +Depth: 2 + +------------------------------------------------------------ + +University Village Pricing and Floor Plans +Pricing +Floor Plan1 Bed/Bath Square Feet1 Price Per Bed, Per Semester2 Price Per Bed +SUMMER 2026 ONLY3 +Efficiency Efficiency/1 350 $4,338 $2,854 +1×1 A 1/1 440 $4,725 $3,109 +1×1 B 1/1 630 $5,150 $3,388 +1×1 C 1/1 530 $5,150 $3,388 +1×1 D 1/1 650 $5,325 $3,503 +2×2 A 2/2 870 $3,825 $2,516 +2×2 B 2/2 975 $4,275 $2,813 +4×2 4/2 1120 $3,250 $2,138 +University Village Pricing and Floor Plans +1 Size information of floor plan is average for typical apartment. Exact dimensions, square footage and layouts vary. +2 The following rates are planned for on-campus University Housing options for the 2025-2026 academic year. Rates are subject to change. Price includes all utilities. +3 The following rates are planned for on-campus University Housing options for summer 2026. Summer rates are only applicable to students who will have a fall 2026 and spring 2027 agreement; therefore, no summer only housing is permitted. Rates are subject to change. Price includes all utilities. +Floor Plans + +Apartment is unfurnished. Measurements are approximate. Floor plans may vary. + +Efficiency +1×1 A — One Bedroom, One Bath +1×1 B — One Bedroom, One Bath +1×1 C — One Bedroom, One Bath +1×1 D — One Bedroom, One Bath +2×2 A – Two Bedroom, Two Bath +2×2 B – Two Bedroom, Two Bath +4×2 – Four Bedroom, Two Bath +RELATED LINKS + +Features & Amenities + +Resident Handbook + +Cancellation Policy (pdf) + +Housing Agreements + +All housing agreements are for the complete academic year. + +Students who live in University Village will sign an academic year agreement (mid-August through mid-May), with an option to participate in summer housing if desired. + +All housing fees associated with assignments in these new buildings will be processed through the Bursar Office. Housing fees for the full housing agreement period will be divided into two equal charges. Your University account will be charged once prior to the fall semester and once prior to the spring semester, in accordance with Bursar timelines. Summer charges for University Commons will be in addition to the listed rates. + +Students who move on campus prior to census day will be charged the full amount of housing for the semester. Charges will not be pro-rated until these dates: + +Spring 2026: Feb. 4 +Summer 2026: June 16 +Fall 2026: Sept. 9 +Spring 2027: Feb. 3 \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..0771502 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,48 @@ +# API in Docker; the Next.js frontend runs on the host with `npm run dev`. +# +# docker compose up --build api # start the API +# docker compose run --rm api python scripts/seed_catalog.py # seed (once) +# +# Optional Ollama container instead of a host install: +# DOCKER_OLLAMA_BASE_URL=http://ollama:11434 docker compose --profile ollama up --build +# docker compose exec ollama ollama pull llama3.1:8b +# docker compose exec ollama ollama pull nomic-embed-text + +services: + api: + build: ./backend + env_file: + - path: ./backend/.env + required: false + environment: + # Container paths override the host-relative defaults in backend/.env. + DATABASE_URL: sqlite:////var/lib/asktemoc/asktemoc.db + CHROMA_PERSIST_DIRECTORY: /var/lib/asktemoc/chroma_db + CATALOG_DATA_DIR: /data/catalog + # Reach Ollama on the host by default (localhost inside the container is + # the container itself). + OLLAMA_BASE_URL: ${DOCKER_OLLAMA_BASE_URL:-http://host.docker.internal:11434} + ports: + - "8000:8000" + volumes: + - api-data:/var/lib/asktemoc + - ./data:/data:ro + extra_hosts: + - "host.docker.internal:host-gateway" + healthcheck: + test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://localhost:8000/api/health')"] + interval: 10s + timeout: 5s + retries: 5 + + ollama: + image: ollama/ollama + profiles: ["ollama"] + ports: + - "11434:11434" + volumes: + - ollama-models:/root/.ollama + +volumes: + api-data: + ollama-models: diff --git a/docs/KNOWN_ISSUES.md b/docs/KNOWN_ISSUES.md new file mode 100644 index 0000000..5b4476e --- /dev/null +++ b/docs/KNOWN_ISSUES.md @@ -0,0 +1,108 @@ +# Known issues + +These were noticed while building the runnable baseline and deliberately left +for the 10-week plan, because they don't block cloning, running and getting +an answer. They're grouped by area and roughly by priority within each group. + +## Security + +- **No sign-in.** Anyone who can reach the web app or the API can chat, and the + API's cost is unbounded. App-level sign-in is the 10-week plan's Weeks 2-4 + assignment. Until then, the deployed web app relies on Vercel Deployment + Protection, and the API on its URL not being shared. An earlier + implementation (Auth.js plus API tokens and a users table) is preserved on the + `reference/auth-oauth` branch. +- **UTD email is Microsoft 365, not Google.** Google OAuth restricted to + utdallas.edu won't work for most students, who have no Google account on + that domain. +- **Admin routes are open too.** `/api/documents/*`, `/api/chroma/*` (including + `DELETE /api/chroma/vectors`) and `/api/dashboard/*` let anyone create, + modify or delete records and vectors. Put them behind auth plus an admin + check, or remove them, before the API is exposed beyond the team. +- **`POST /api/query/` is a placeholder.** It ignores the question and asks a + hard-coded Ollama model for the capital of a random country. +- **No rate limiting or quotas** on `/api/chat`. +- **Placeholder 401/403 handling.** In `frontend/src/lib/api.ts`, the 401/403 + branch only follows a `Location` header. It's replaced as part of the auth + work. + +## Backend divergence + +- **Production may run AskTemoc_Backend's `speedrun` branch, not `main`.** + `speedrun` is 12 commits ahead of `main`. It has a citation prompt producing + `[n, "verbatim quote"]`, a lower temperature, and a different SSE shape + (`delta` plus cumulative `message`, and `error` events). This repo imported + `main`. The web app accepts both SSE shapes, but the `[n, "…"]` citation pills + in `src/lib/citations.ts` only appear with the `speedrun` prompt; with + `main`'s prompt, answers cite sources in prose. Decide what to port. +- Other unmerged AskTemoc_Backend branches: `citations-test`, + `feat/update_scrapers` (scraper reorganisation and change detection), + `db_sync` (Streamlit dashboard), `VectorService`. + +## Retrieval and data + +- **Repeated sources.** The retriever returns 4 chunks, often from the same + program page, so the UI can list the same URL several times. +- **Corpus noise.** The scraped text includes page chrome ("Download page as a + PDF", "Compare Versions"). Folders `data/catalog/1`, `2`, `3` are artefacts of + footnote links and have no catalog page, so they cite the programs index. + The catalog year (2025) is hard-coded in the scraper and seed script. +- **Ambiguous URLs.** `bachelor_of_science_in_computational_and_geospatial_science` + and `bachelor_of_science_in_data_science` are each linked twice from the + catalog index; `sources.json` keeps the first link. +- **Housing isn't seeded.** `data/housing/` (from the backend repo's housing + scraper) isn't seeded. +- **Embedding/model mismatch only warns.** If the collection was seeded with a + different provider or model, the API logs a warning but still serves + (wrong) results. `seed_catalog.py` rebuilds the collection when it detects + the mismatch. +- **Chroma is a local directory.** That's fine for one dev machine or one + container with a volume, but Cloud Run's filesystem is ephemeral. Production + needs persistent storage or the planned pgvector migration. +- **The SQLite Document/Chunk/Embedding tables aren't the retrieval store.** + The seed script writes straight to Chroma, so these tables (and the + `/api/documents` and `/api/chroma` export flow) are unused by chat. + +## Backend code + +- `DataIngestionService` (`app/services/data_ingestion_service.py`) is + superseded by `scripts/seed_catalog.py`. It only reads `requirements.txt`, + adds duplicates on re-run, and uses folder names as sources. +- `general_webscraper.py`, `html_processing_pipeline.py`, `housing_scraper.py` + and `scraper_service.py` import Crawl4AI, docling or Playwright. They only + work with `requirements-scrape.txt` installed, and nothing in the API imports + them. `embedding_service.py` defines a separate FastAPI app using + HuggingFace embeddings and is unused. +- **No migrations.** Tables are created with `Base.metadata.create_all`, so + later column changes won't apply to existing databases (add Alembic before + the Postgres move). The Postgres driver isn't in `requirements.txt`. +- **Deprecations:** `@app.on_event("startup")` (use lifespan), + `datetime.utcnow`, Pydantic v1-style `class Config` and `min_items` in + `app/schemas/db_schemas.py`. These show as warnings in pytest. +- **Errors look like answers.** A failure mid-stream is reported as a `text` + event, "Error processing request", followed by `done`, so the UI shows it as + answer text. A dedicated `error` event would be clearer. +- **The Docker image** runs as root and keeps `build-essential`. + +## Frontend + +- **npm 11 skips install scripts** for `sharp`, `@tailwindcss/oxide` and + `unrs-resolver` (the "allow-scripts" warnings). Lint and build pass without + them; run `npm approve-scripts` if image optimisation misbehaves. +- **`next start` locally talks to production.** It loads + `frontend/.env.production`, which points `ASKTEMOC_API_ORIGIN` at Cloud Run. + Set `ASKTEMOC_API_ORIGIN=http://localhost:8000` to test a production build + against a local API. `npm run dev` proxies to the local API by default. +- **Conversations aren't saved.** They live in React state, with three + hard-coded sample conversations, and a reload clears them. (Persistence is + part of the 10-week plan.) +- **The landing page's "Learn More" button links to `/`.** + +## Repo hygiene + +- **Stale backend docs.** `backend/IMPLEMENTATION_STATUS.md`, + `IMPLEMENTATION_SUMMARY.md`, `CHANGES.md`, `QUICK_START.md` and + `DATABASE_API_DOCS.md` describe the old standalone backend and were left + unchanged on purpose. +- **Retire the old repo.** `davis118/AskTemoc_Backend` should be archived or + pointed here once this branch merges, so work doesn't split again. diff --git a/frontend/.env.example b/frontend/.env.example index 9a5e093..8ef6b36 100644 --- a/frontend/.env.example +++ b/frontend/.env.example @@ -1,22 +1,20 @@ -# Vercel: run `vercel link` / `vercel deploy` from the repo root, not from -# frontend/, so Root Directory "frontend" matches once (avoids frontend/frontend). - -# Local dev: backend on your machine -NEXT_PUBLIC_API_BASE_URL=http://localhost:8000 +# Copy to frontend/.env.local only if you need to override a default; the app +# runs locally without any env vars. -# Production on Vercel: omit NEXT_PUBLIC_API_BASE_URL so the app uses same-origin -# /api/proxy (see next.config rewrites). Never set it to http://localhost — the -# browser would call the user’s machine, not Cloud Run. +# Where the same-origin /api/proxy rewrites (next.config.ts) forward to. +# `next dev` defaults to the local API at http://localhost:8000; production builds +# use the Cloud Run origin from .env.production and vercel.json. +# ASKTEMOC_API_ORIGIN=http://localhost:8000 -# Used at build time for rewrites (also in .env.production and vercel.json env). -# ASKTEMOC_API_ORIGIN=https://asktemoc-api-422855032073.us-central1.run.app +# Optional: call an API origin directly from the browser instead of the proxy +# (requires CORS on that API). Never set it to http://localhost in Production: +# the browser would call the visitor's machine, not Cloud Run. +# NEXT_PUBLIC_API_BASE_URL=http://localhost:8000 -# Optional: call Cloud Run directly from the browser (requires CORS on the API). -# NEXT_PUBLIC_API_BASE_URL=https://asktemoc-api-422855032073.us-central1.run.app - -# Vercel Dashboard → Environment Variables: for Production/Preview, delete any -# NEXT_PUBLIC_API_BASE_URL=http://localhost... unless you truly mean it. +# Vercel: run `vercel link` / `vercel deploy` from the repo root, not from +# frontend/, so Root Directory "frontend" matches once (avoids frontend/frontend). -# Browser login for visitors: Vercel Dashboard → Project → Deployment Protection → -# enable “Vercel Authentication” (or Password on Pro). Unauthenticated users are -# redirected to Vercel’s sign-in page before the app loads. +# Browser login for visitors: Vercel Dashboard -> Project -> Deployment Protection -> +# enable "Vercel Authentication" (or Password on Pro). Unauthenticated users are +# redirected to Vercel's sign-in page before the app loads. App-level sign-in is +# planned (10-week plan, Weeks 2-4). diff --git a/frontend/README.md b/frontend/README.md index 9c8817d..ef61736 100644 --- a/frontend/README.md +++ b/frontend/README.md @@ -1,42 +1,63 @@ -This is a [Next.js](https://nextjs.org) project bootstrapped with [`create-next-app`](https://nextjs.org/docs/app/api-reference/cli/create-next-app). +# Ask Temoc — Web App -## Getting Started +Next.js 15 / React 19 chat UI. For setup, see the [root README](../README.md). -First, install the dependencies: +## Scripts -```bash -npm install -``` +- `npm run dev`: development server on http://localhost:3000 +- `npm run build` / `npm start`: production build and server +- `npm run lint`: ESLint -Then, run the development server: +## How chat reaches the API -```bash -npm run dev -# or -yarn dev -# or -pnpm dev -# or -bun dev -``` +The browser calls same-origin `/api/proxy/chat` and `/api/proxy/health`. +`next.config.ts` rewrites those to `ASKTEMOC_API_ORIGIN`: -Open [http://localhost:3000](http://localhost:3000) with your browser to see the result. +- `npm run dev`: `http://localhost:8000` by default (the local API). +- Production builds and Vercel: the Cloud Run origin, set in `.env.production` + and `vercel.json`. -You can start editing the page by modifying `app/page.tsx`. The page auto-updates as you edit the file. +`NEXT_PUBLIC_API_BASE_URL` makes the browser call an API origin directly instead +of going through the proxy. That needs CORS on the API; never point it at +localhost in production. -This project uses [`next/font`](https://nextjs.org/docs/app/building-your-application/optimizing/fonts) to automatically optimize and load [Geist](https://vercel.com/font), a new font family for Vercel. +There's no app-level sign-in yet; it's planned for Weeks 2-4 of the 10-week plan. +In production, Vercel Deployment Protection gates visitors. -## Learn More +## Response format -To learn more about Next.js, take a look at the following resources: +SSE events, one `data: {json}` per line: -- [Next.js Documentation](https://nextjs.org/docs) - learn about Next.js features and API. -- [Learn Next.js](https://nextjs.org/learn) - an interactive Next.js tutorial. +- `source`: `message` is `Source n: `, plus `index`, `url` and optionally + `title`. +- `text`: the next piece of the answer. It's appended; when an event also has a + `delta` field (older API builds), the `delta` is appended instead. +- `done`: end of stream. Unknown types are ignored. -You can check out [the Next.js GitHub repository](https://github.com/vercel/next.js) - your feedback and contributions are welcome! +Assistant text may embed citations as `[n, "exact quoted text"]`, which +`src/lib/citations.ts` turns into linked pills. -## Deploy on Vercel +## Source layout + +``` +src/ + app/ + page.tsx landing page + layout.tsx root layout (Geist fonts via next/font, navbar) + chat/ chat page, sidebar, message area, citation rendering + components/ navbar and shadcn/ui primitives + lib/ + api.ts chat API client + SSE parsing + citations.ts [n, "…"] citation parsing and source resolution + markdown.ts inline markdown (**bold**, *italic*, `code`) + types/ shared Message/Conversation types +``` -The easiest way to deploy your Next.js app is to use the [Vercel Platform](https://vercel.com/new?utm_medium=default-template&filter=next.js&utm_source=create-next-app&utm_campaign=create-next-app-readme) from the creators of Next.js. +Conversations are held in React state only, so a reload clears them. + +## Deploy on Vercel -Check out our [Next.js deployment documentation](https://nextjs.org/docs/app/building-your-application/deploying) for more details. +Run `vercel link` / `vercel deploy` from the **repository root**, not from +`frontend/`, with the project's Root Directory set to `frontend`. To require a +login before the app loads, enable Vercel Authentication under Project → +Deployment Protection. diff --git a/frontend/next.config.ts b/frontend/next.config.ts index 2133055..c70e7b5 100644 --- a/frontend/next.config.ts +++ b/frontend/next.config.ts @@ -1,8 +1,9 @@ import type { NextConfig } from "next"; +// Where /api/proxy/* is forwarded. `next dev` defaults to the local API; production +// builds get the Cloud Run origin from .env.production / vercel.json. const ASKTEMOC_API_ORIGIN = - process.env.ASKTEMOC_API_ORIGIN?.replace(/\/$/, "") || - "https://asktemoc-api-422855032073.us-central1.run.app"; + process.env.ASKTEMOC_API_ORIGIN?.replace(/\/$/, "") || "http://localhost:8000"; const nextConfig: NextConfig = { async rewrites() { diff --git a/frontend/package-lock.json b/frontend/package-lock.json index 4518339..e3bae12 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -17,7 +17,7 @@ "class-variance-authority": "^0.7.1", "clsx": "^2.1.1", "lucide-react": "^0.544.0", - "next": "^15.5.15", + "next": "^15.5.26", "react": "19.1.0", "react-dom": "19.1.0", "tailwind-merge": "^3.3.1" @@ -29,7 +29,7 @@ "@types/react": "^19", "@types/react-dom": "^19", "eslint": "^9", - "eslint-config-next": "15.5.7", + "eslint-config-next": "^15.5.26", "tailwindcss": "^4", "tw-animate-css": "^1.3.8", "typescript": "^5" @@ -807,15 +807,15 @@ } }, "node_modules/@next/env": { - "version": "15.5.15", - "resolved": "https://registry.npmjs.org/@next/env/-/env-15.5.15.tgz", - "integrity": "sha512-vcmyu5/MyFzN7CdqRHO3uHO44p/QPCZkuTUXroeUmhNP8bL5PHFEhik22JUazt+CDDoD6EpBYRCaS2pISL+/hg==", + "version": "15.5.26", + "resolved": "https://registry.npmjs.org/@next/env/-/env-15.5.26.tgz", + "integrity": "sha512-NJBz9q10LU9h3KjHLEbdgWIV+ow/x+MYzKBRfqhm9/QmML3tPMhYmXF/UIV9SDVCNtOqFNc5oX7kZqeiigMCEA==", "license": "MIT" }, "node_modules/@next/eslint-plugin-next": { - "version": "15.5.7", - "resolved": "https://registry.npmjs.org/@next/eslint-plugin-next/-/eslint-plugin-next-15.5.7.tgz", - "integrity": "sha512-DtRU2N7BkGr8r+pExfuWHwMEPX5SD57FeA6pxdgCHODo+b/UgIgjE+rgWKtJAbEbGhVZ2jtHn4g3wNhWFoNBQQ==", + "version": "15.5.26", + "resolved": "https://registry.npmjs.org/@next/eslint-plugin-next/-/eslint-plugin-next-15.5.26.tgz", + "integrity": "sha512-xeVpG8CbnTf9IelaFbZQ7TN4BXHV2x+ejnXXbY7Z4kD+sgJ8j97psraV+4/fGjLCjNQuEofUHMeIeieDMyKO/w==", "dev": true, "license": "MIT", "dependencies": { @@ -823,9 +823,9 @@ } }, "node_modules/@next/swc-darwin-arm64": { - "version": "15.5.15", - "resolved": "https://registry.npmjs.org/@next/swc-darwin-arm64/-/swc-darwin-arm64-15.5.15.tgz", - "integrity": "sha512-6PvFO2Tzt10GFK2Ro9tAVEtacMqRmTarYMFKAnV2vYMdwWc73xzmDQyAV7SwEdMhzmiRoo7+m88DuiXlJlGeaw==", + "version": "15.5.26", + "resolved": "https://registry.npmjs.org/@next/swc-darwin-arm64/-/swc-darwin-arm64-15.5.26.tgz", + "integrity": "sha512-So8eoJxIcXw/TexNUvvh3uY72J9nDo5BpJsAwUKx+FK57CrWXg6RqVufV7U9OT3BO+siMzJ2FuAwBhaHoPlLGg==", "cpu": [ "arm64" ], @@ -839,9 +839,9 @@ } }, "node_modules/@next/swc-darwin-x64": { - "version": "15.5.15", - "resolved": "https://registry.npmjs.org/@next/swc-darwin-x64/-/swc-darwin-x64-15.5.15.tgz", - "integrity": "sha512-G+YNV+z6FDZTp/+IdGyIMFqalBTaQSnvAA+X/hrt+eaTRFSznRMz9K7rTmzvM6tDmKegNtyzgufZW0HwVzEqaQ==", + "version": "15.5.26", + "resolved": "https://registry.npmjs.org/@next/swc-darwin-x64/-/swc-darwin-x64-15.5.26.tgz", + "integrity": "sha512-jImzLUTClVWKhP91e5sgDumjxCLhaFSt7DuN5cnRYw99Dppxxhhq5jKRyDa2aTv3JE7dQmTSTsY3EFkSOp9Pog==", "cpu": [ "x64" ], @@ -855,9 +855,9 @@ } }, "node_modules/@next/swc-linux-arm64-gnu": { - "version": "15.5.15", - "resolved": "https://registry.npmjs.org/@next/swc-linux-arm64-gnu/-/swc-linux-arm64-gnu-15.5.15.tgz", - "integrity": "sha512-eVkrMcVIBqGfXB+QUC7jjZ94Z6uX/dNStbQFabewAnk13Uy18Igd1YZ/GtPRzdhtm7QwC0e6o7zOQecul4iC1w==", + "version": "15.5.26", + "resolved": "https://registry.npmjs.org/@next/swc-linux-arm64-gnu/-/swc-linux-arm64-gnu-15.5.26.tgz", + "integrity": "sha512-CaWd+T/Lud2BmZbrsa1CzCnIOdU3YX9Nuk89virZaSB1O+C+8Yrrevgmnl68u4dvZIfOAzQ77S+Njrq7v1XwSA==", "cpu": [ "arm64" ], @@ -871,9 +871,9 @@ } }, "node_modules/@next/swc-linux-arm64-musl": { - "version": "15.5.15", - "resolved": "https://registry.npmjs.org/@next/swc-linux-arm64-musl/-/swc-linux-arm64-musl-15.5.15.tgz", - "integrity": "sha512-RwSHKMQ7InLy5GfkY2/n5PcFycKA08qI1VST78n09nN36nUPqCvGSMiLXlfUmzmpQpF6XeBYP2KRWHi0UW3uNg==", + "version": "15.5.26", + "resolved": "https://registry.npmjs.org/@next/swc-linux-arm64-musl/-/swc-linux-arm64-musl-15.5.26.tgz", + "integrity": "sha512-97AyKI34yjpaudlkWHswAf7c1PjWQAC7lLyrw3R5K+bq834EOA+9IG68rIVy0VqrqGjtPSMjJmMmgeJ3wHR5og==", "cpu": [ "arm64" ], @@ -887,9 +887,9 @@ } }, "node_modules/@next/swc-linux-x64-gnu": { - "version": "15.5.15", - "resolved": "https://registry.npmjs.org/@next/swc-linux-x64-gnu/-/swc-linux-x64-gnu-15.5.15.tgz", - "integrity": "sha512-nplqvY86LakS+eeiuWsNWvfmK8pFcOEW7ZtVRt4QH70lL+0x6LG/m1OpJ/tvrbwjmR8HH9/fH2jzW1GlL03TIg==", + "version": "15.5.26", + "resolved": "https://registry.npmjs.org/@next/swc-linux-x64-gnu/-/swc-linux-x64-gnu-15.5.26.tgz", + "integrity": "sha512-eVtuOCew1sBPV7BEgxy7qxuVyqoU3tJIS/xDZwa1/NiQ4Q0LM2JMH7rny+/uVIN6hsQ3PTb0hTQWypA0L2QxtA==", "cpu": [ "x64" ], @@ -903,9 +903,9 @@ } }, "node_modules/@next/swc-linux-x64-musl": { - "version": "15.5.15", - "resolved": "https://registry.npmjs.org/@next/swc-linux-x64-musl/-/swc-linux-x64-musl-15.5.15.tgz", - "integrity": "sha512-eAgl9NKQ84/sww0v81DQINl/vL2IBxD7sMybd0cWRw6wqgouVI53brVRBrggqBRP/NWeIAE1dm5cbKYoiMlqDQ==", + "version": "15.5.26", + "resolved": "https://registry.npmjs.org/@next/swc-linux-x64-musl/-/swc-linux-x64-musl-15.5.26.tgz", + "integrity": "sha512-EiUXADp+Z+OdQnSqbX10YgOSUrs0CXVODoTfySfsP2jdhngn9bq5RJd376FJzqMPe/XX25FMr1aXtYUVPA0qDw==", "cpu": [ "x64" ], @@ -919,9 +919,9 @@ } }, "node_modules/@next/swc-win32-arm64-msvc": { - "version": "15.5.15", - "resolved": "https://registry.npmjs.org/@next/swc-win32-arm64-msvc/-/swc-win32-arm64-msvc-15.5.15.tgz", - "integrity": "sha512-GJVZC86lzSquh0MtvZT+L7G8+jMnJcldloOjA8Kf3wXvBrvb6OGe2MzPuALxFshSm/IpwUtD2mIoof39ymf52A==", + "version": "15.5.26", + "resolved": "https://registry.npmjs.org/@next/swc-win32-arm64-msvc/-/swc-win32-arm64-msvc-15.5.26.tgz", + "integrity": "sha512-HPl41fgkC4kdM5CCIoqNW6KlKEj1N+xS6bjNFFxDNKooUNUu09frgD948zo95TPw/C3XINZpkdkBGNU2RhjEnw==", "cpu": [ "arm64" ], @@ -935,9 +935,9 @@ } }, "node_modules/@next/swc-win32-x64-msvc": { - "version": "15.5.15", - "resolved": "https://registry.npmjs.org/@next/swc-win32-x64-msvc/-/swc-win32-x64-msvc-15.5.15.tgz", - "integrity": "sha512-nFucjVdwlFqxh/JG3hWSJ4p8+YJV7Ii8aPDuBQULB6DzUF4UNZETXLfEUk+oI2zEznWWULPt7MeuTE6xtK1HSA==", + "version": "15.5.26", + "resolved": "https://registry.npmjs.org/@next/swc-win32-x64-msvc/-/swc-win32-x64-msvc-15.5.26.tgz", + "integrity": "sha512-TgmJ5ginKr34RPsz01/swpYtFBxh51d66jM26aytpE6NIypU5KtBVI8C2hJjUNpWr7v6sWj8a6+og2MyntcnpA==", "cpu": [ "x64" ], @@ -3529,13 +3529,13 @@ } }, "node_modules/eslint-config-next": { - "version": "15.5.7", - "resolved": "https://registry.npmjs.org/eslint-config-next/-/eslint-config-next-15.5.7.tgz", - "integrity": "sha512-nU/TRGHHeG81NeLW5DeQT5t6BDUqbpsNQTvef1ld/tqHT+/zTx60/TIhKnmPISTTe++DVo+DLxDmk4rnwHaZVw==", + "version": "15.5.26", + "resolved": "https://registry.npmjs.org/eslint-config-next/-/eslint-config-next-15.5.26.tgz", + "integrity": "sha512-nekKvDmRQV8VRUhBTt2B/HFdgNR0vWKA/ao5W1L7lcaEvAXsuQYjIJqwLUGaux9uvXOrxnxLu5dQjnRnOixcfw==", "dev": true, "license": "MIT", "dependencies": { - "@next/eslint-plugin-next": "15.5.7", + "@next/eslint-plugin-next": "15.5.26", "@rushstack/eslint-patch": "^1.10.3", "@typescript-eslint/eslint-plugin": "^5.4.2 || ^6.0.0 || ^7.0.0 || ^8.0.0", "@typescript-eslint/parser": "^5.4.2 || ^6.0.0 || ^7.0.0 || ^8.0.0", @@ -5357,12 +5357,12 @@ "license": "MIT" }, "node_modules/next": { - "version": "15.5.15", - "resolved": "https://registry.npmjs.org/next/-/next-15.5.15.tgz", - "integrity": "sha512-VSqCrJwtLVGwAVE0Sb/yikrQfkwkZW9p+lL/J4+xe+G3ZA+QnWPqgcfH1tDUEuk9y+pthzzVFp4L/U8JerMfMQ==", + "version": "15.5.26", + "resolved": "https://registry.npmjs.org/next/-/next-15.5.26.tgz", + "integrity": "sha512-EVCqhvq8Hs+nX9udH2VzE/iXAg9QodZBZnwVJTuAMl386GIYvlJtYhFytV9nSlDYxKw3kEyv8I2dCQs0+on0sQ==", "license": "MIT", "dependencies": { - "@next/env": "15.5.15", + "@next/env": "15.5.26", "@swc/helpers": "0.5.15", "caniuse-lite": "^1.0.30001579", "postcss": "8.4.31", @@ -5375,15 +5375,15 @@ "node": "^18.18.0 || ^19.8.0 || >= 20.0.0" }, "optionalDependencies": { - "@next/swc-darwin-arm64": "15.5.15", - "@next/swc-darwin-x64": "15.5.15", - "@next/swc-linux-arm64-gnu": "15.5.15", - "@next/swc-linux-arm64-musl": "15.5.15", - "@next/swc-linux-x64-gnu": "15.5.15", - "@next/swc-linux-x64-musl": "15.5.15", - "@next/swc-win32-arm64-msvc": "15.5.15", - "@next/swc-win32-x64-msvc": "15.5.15", - "sharp": "^0.34.3" + "@next/swc-darwin-arm64": "15.5.26", + "@next/swc-darwin-x64": "15.5.26", + "@next/swc-linux-arm64-gnu": "15.5.26", + "@next/swc-linux-arm64-musl": "15.5.26", + "@next/swc-linux-x64-gnu": "15.5.26", + "@next/swc-linux-x64-musl": "15.5.26", + "@next/swc-win32-arm64-msvc": "15.5.26", + "@next/swc-win32-x64-msvc": "15.5.26", + "sharp": "^0.34.3 || ^0.35.4" }, "peerDependencies": { "@opentelemetry/api": "^1.1.0", diff --git a/frontend/package.json b/frontend/package.json index 641b697..9464c50 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -18,7 +18,7 @@ "class-variance-authority": "^0.7.1", "clsx": "^2.1.1", "lucide-react": "^0.544.0", - "next": "^15.5.15", + "next": "^15.5.26", "react": "19.1.0", "react-dom": "19.1.0", "tailwind-merge": "^3.3.1" @@ -30,7 +30,7 @@ "@types/react": "^19", "@types/react-dom": "^19", "eslint": "^9", - "eslint-config-next": "15.5.7", + "eslint-config-next": "^15.5.26", "tailwindcss": "^4", "tw-animate-css": "^1.3.8", "typescript": "^5" diff --git a/frontend/src/app/chat/assistant-text.tsx b/frontend/src/app/chat/assistant-text.tsx index 9a096e8..a3cdad0 100644 --- a/frontend/src/app/chat/assistant-text.tsx +++ b/frontend/src/app/chat/assistant-text.tsx @@ -13,6 +13,7 @@ import { type SourceMeta, splitInlineCitations, } from "@/lib/citations"; +import { splitInlineMarkdown } from "@/lib/markdown"; import type { Message } from "@/types"; const PILL_LABEL_MAX = 80; @@ -86,6 +87,36 @@ function CitationPill({ ); } +function InlineMarkdown({ text }: { text: string }) { + return ( + <> + {splitInlineMarkdown(text).map((seg, i) => { + switch (seg.type) { + case "strong": + return ( + + {seg.text} + + ); + case "em": + return {seg.text}; + case "code": + return ( + + {seg.text} + + ); + default: + return {seg.text}; + } + })} + + ); +} + export function AssistantFormattedText({ text, content, @@ -100,7 +131,7 @@ export function AssistantFormattedText({ <> {segments.map((seg, i) => seg.type === "text" ? ( - {seg.text} + ) : ( { + const handleKeyDown = (e: React.KeyboardEvent) => { if (e.key === "Enter" && !e.shiftKey) { e.preventDefault(); sendMessage(); @@ -259,7 +260,8 @@ export default function ChatArea({ {!isBackendAvailable && (

)} @@ -268,14 +270,14 @@ export default function ChatArea({ setInputValue(e.target.value)} - onKeyPress={handleKeyPress} - placeholder={isBackendAvailable ? "Ask Temoc anything about UTD..." : "Backend unavailable..."} - disabled={!isBackendAvailable || isStreaming} + onKeyDown={handleKeyDown} + placeholder="Ask Temoc anything about UTD..." + disabled={isStreaming} className="flex-1 border-utd-primary/30 focus:border-utd-primary focus:ring-utd-primary/20" />