diff --git a/.github/workflows/android-build.yml b/.github/workflows/android-build.yml new file mode 100644 index 0000000..8572355 --- /dev/null +++ b/.github/workflows/android-build.yml @@ -0,0 +1,56 @@ +name: Android build + +on: + pull_request: + paths: + - "apps/android/**" + - "gradle/**" + - "gradlew" + - "gradlew.bat" + - "settings.gradle.kts" + - "build.gradle.kts" + - ".github/workflows/android-build.yml" + push: + branches: [main] + paths: + - "apps/android/**" + - "gradle/**" + - "gradlew" + - "gradlew.bat" + - "settings.gradle.kts" + - "build.gradle.kts" + - ".github/workflows/android-build.yml" + workflow_dispatch: + +permissions: + contents: read + +jobs: + build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Detect Android application + id: android + run: | + if [ -f gradlew ] && [ -d apps/android ]; then + echo "enabled=true" >> "$GITHUB_OUTPUT" + else + echo "enabled=false" >> "$GITHUB_OUTPUT" + echo "Android application is not implemented yet; build skipped." + fi + - uses: actions/setup-java@v4 + if: steps.android.outputs.enabled == 'true' + with: + distribution: temurin + java-version: 21 + cache: gradle + - uses: gradle/actions/setup-gradle@v4 + if: steps.android.outputs.enabled == 'true' + - run: ./gradlew :apps:android:lintDebug :apps:android:testDebugUnitTest :apps:android:assembleDebug + if: steps.android.outputs.enabled == 'true' + - uses: actions/upload-artifact@v4 + if: steps.android.outputs.enabled == 'true' + with: + name: rockql-android-debug + path: apps/android/build/outputs/apk/debug/*.apk diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..f3ca5bf --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,23 @@ +name: CI + +on: + pull_request: + push: + branches: [main] + +permissions: + contents: read + +jobs: + rust: + name: Rust quality + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@stable + with: + components: rustfmt, clippy + - name: Formatting + run: cargo fmt --all -- --check + - name: Clippy + run: cargo clippy --workspace --all-targets -- -D warnings diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml new file mode 100644 index 0000000..2017d84 --- /dev/null +++ b/.github/workflows/docs.yml @@ -0,0 +1,45 @@ +name: Documentation + +on: + pull_request: + paths: + - "README.md" + - "docs/**" + - "CONTRIBUTING.md" + - "CODE_OF_CONDUCT.md" + - "SECURITY.md" + - ".github/workflows/docs.yml" + push: + branches: [main] + paths: + - "README.md" + - "docs/**" + - "CONTRIBUTING.md" + - "CODE_OF_CONDUCT.md" + - "SECURITY.md" + - ".github/workflows/docs.yml" + +permissions: + contents: read + +jobs: + quality: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Check Markdown whitespace + run: | + python - <<'PY' + from pathlib import Path + files = [Path("README.md"), Path("CONTRIBUTING.md"), Path("CODE_OF_CONDUCT.md"), Path("SECURITY.md")] + files.extend(Path("docs").rglob("*.md")) + failures = [] + for path in files: + for number, line in enumerate(path.read_text(encoding="utf-8").splitlines(), 1): + if line != line.rstrip(): + failures.append(f"{path}:{number}: trailing whitespace") + if "\t" in line: + failures.append(f"{path}:{number}: tab character") + if failures: + raise SystemExit("\n".join(failures)) + PY diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..e3d95af --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,74 @@ +name: Release + +on: + push: + tags: ["v*"] + workflow_dispatch: + +permissions: + contents: write + +jobs: + build: + strategy: + fail-fast: false + matrix: + include: + - os: ubuntu-latest + target: x86_64-unknown-linux-gnu + artifact: rockql-linux-x64 + binary: rockql + - os: ubuntu-24.04-arm + target: aarch64-unknown-linux-gnu + artifact: rockql-linux-arm64 + binary: rockql + - os: windows-latest + target: x86_64-pc-windows-msvc + artifact: rockql-windows-x64 + binary: rockql.exe + - os: macos-13 + target: x86_64-apple-darwin + artifact: rockql-macos-x64 + binary: rockql + - os: macos-14 + target: aarch64-apple-darwin + artifact: rockql-macos-arm64 + binary: rockql + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@stable + with: + targets: ${{ matrix.target }} + - run: cargo build --release --target ${{ matrix.target }} -p rockql-cli + - name: Stage binary + shell: python + run: | + import pathlib + import shutil + source = pathlib.Path("target") / "${{ matrix.target }}" / "release" / "${{ matrix.binary }}" + destination = pathlib.Path("dist") / "${{ matrix.artifact }}" / "${{ matrix.binary }}" + destination.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(source, destination) + - uses: actions/upload-artifact@v4 + with: + name: ${{ matrix.artifact }} + path: dist/${{ matrix.artifact }} + + publish: + if: startsWith(github.ref, 'refs/tags/') + needs: build + runs-on: ubuntu-latest + steps: + - uses: actions/download-artifact@v4 + with: + path: dist + - name: Generate checksums + run: find dist -type f -exec sha256sum {} + > dist/SHA256SUMS + - uses: softprops/action-gh-release@v2 + with: + generate_release_notes: true + files: | + dist/**/rockql + dist/**/rockql.exe + dist/SHA256SUMS diff --git a/.github/workflows/rust-tests.yml b/.github/workflows/rust-tests.yml new file mode 100644 index 0000000..78722e2 --- /dev/null +++ b/.github/workflows/rust-tests.yml @@ -0,0 +1,29 @@ +name: Rust tests + +on: + pull_request: + paths: + - "Cargo.toml" + - "Cargo.lock" + - "compiler/**" + - "examples/**" + - ".github/workflows/rust-tests.yml" + push: + branches: [main] + paths: + - "Cargo.toml" + - "Cargo.lock" + - "compiler/**" + - "examples/**" + - ".github/workflows/rust-tests.yml" + +permissions: + contents: read + +jobs: + test: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@stable + - run: cargo test --workspace --all-targets diff --git a/.github/workflows/security.yml b/.github/workflows/security.yml new file mode 100644 index 0000000..70f1e0d --- /dev/null +++ b/.github/workflows/security.yml @@ -0,0 +1,45 @@ +name: Security + +on: + pull_request: + schedule: + - cron: "17 3 * * 1" + workflow_dispatch: + +permissions: + contents: read + +jobs: + dependency-review: + if: github.event_name == 'pull_request' + runs-on: ubuntu-latest + permissions: + contents: read + pull-requests: read + steps: + - uses: actions/checkout@v4 + - name: Review dependency changes + continue-on-error: true + uses: actions/dependency-review-action@v4 + + audit: + if: github.event_name != 'pull_request' + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@stable + - uses: taiki-e/install-action@cargo-deny + - run: cargo deny check advisories + + codeql: + runs-on: ubuntu-latest + permissions: + security-events: write + packages: read + contents: read + steps: + - uses: actions/checkout@v4 + - uses: github/codeql-action/init@v3 + with: + languages: rust + - uses: github/codeql-action/analyze@v3 diff --git a/.github/workflows/wasm-build.yml b/.github/workflows/wasm-build.yml new file mode 100644 index 0000000..1bf9533 --- /dev/null +++ b/.github/workflows/wasm-build.yml @@ -0,0 +1,50 @@ +name: WebAssembly build + +on: + pull_request: + paths: + - "bindings/rockql-js/**" + - "compiler/**" + - "Cargo.toml" + - "Cargo.lock" + - ".github/workflows/wasm-build.yml" + push: + branches: [main] + paths: + - "bindings/rockql-js/**" + - "compiler/**" + - "Cargo.toml" + - "Cargo.lock" + - ".github/workflows/wasm-build.yml" + workflow_dispatch: + +permissions: + contents: read + +jobs: + build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Detect WebAssembly binding + id: wasm + run: | + if [ -f bindings/rockql-js/Cargo.toml ]; then + echo "enabled=true" >> "$GITHUB_OUTPUT" + else + echo "enabled=false" >> "$GITHUB_OUTPUT" + echo "WebAssembly binding is not implemented yet; build skipped." + fi + - uses: dtolnay/rust-toolchain@stable + if: steps.wasm.outputs.enabled == 'true' + with: + targets: wasm32-unknown-unknown + - uses: taiki-e/install-action@wasm-pack + if: steps.wasm.outputs.enabled == 'true' + - run: wasm-pack build bindings/rockql-js --target web --release + if: steps.wasm.outputs.enabled == 'true' + - uses: actions/upload-artifact@v4 + if: steps.wasm.outputs.enabled == 'true' + with: + name: rockql-wasm + path: bindings/rockql-js/pkg diff --git a/.github/workflows/web-build.yml b/.github/workflows/web-build.yml new file mode 100644 index 0000000..b2c41e5 --- /dev/null +++ b/.github/workflows/web-build.yml @@ -0,0 +1,48 @@ +name: Web build + +on: + pull_request: + paths: + - "apps/playground/**" + - "bindings/rockql-js/**" + - "package.json" + - "package-lock.json" + - ".github/workflows/web-build.yml" + push: + branches: [main] + paths: + - "apps/playground/**" + - "bindings/rockql-js/**" + - "package.json" + - "package-lock.json" + - ".github/workflows/web-build.yml" + workflow_dispatch: + +permissions: + contents: read + +jobs: + build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Detect playground + id: playground + run: | + if [ -f package.json ] && [ -d apps/playground ]; then + echo "enabled=true" >> "$GITHUB_OUTPUT" + else + echo "enabled=false" >> "$GITHUB_OUTPUT" + echo "Playground is not implemented yet; build skipped." + fi + - uses: actions/setup-node@v4 + if: steps.playground.outputs.enabled == 'true' + with: + node-version: 22 + cache: npm + - run: npm ci + if: steps.playground.outputs.enabled == 'true' + - run: npm run typecheck --if-present + if: steps.playground.outputs.enabled == 'true' + - run: npm run build + if: steps.playground.outputs.enabled == 'true' diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..70df8dc --- /dev/null +++ b/.gitignore @@ -0,0 +1,8 @@ +/target +**/*.rs.bk +.DS_Store +.idea/ +.vscode/ +node_modules/ +dist/ +.env diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md new file mode 100644 index 0000000..409a09d --- /dev/null +++ b/CODE_OF_CONDUCT.md @@ -0,0 +1,17 @@ +# Contributor Covenant Code of Conduct + +## Our pledge + +We pledge to make participation in RockQL a harassment-free experience for everyone, regardless of age, body size, disability, ethnicity, sex characteristics, gender identity and expression, level of experience, education, socio-economic status, nationality, personal appearance, race, caste, colour, religion, or sexual identity and orientation. + +## Our standards + +Use welcoming language, respect differing viewpoints, accept constructive feedback, focus on what is best for the community, and show empathy towards other contributors. + +Unacceptable behaviour includes harassment, trolling, insulting comments, publishing private information without permission, or other conduct that could reasonably be considered inappropriate in a professional setting. + +## Enforcement + +Report unacceptable behaviour through the repository security or maintainer contact channels. Maintainers may remove, edit, or reject contributions and may temporarily or permanently ban contributors for behaviour they consider inappropriate, threatening, offensive, or harmful. + +This policy is adapted from Contributor Covenant 2.1. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..9b9755e --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,16 @@ +# Contributing to RockQL + +Thank you for helping build RockQL. + +## Development + +1. Install the stable Rust toolchain. +2. Run `cargo fmt --all -- --check`. +3. Run `cargo clippy --workspace --all-targets -- -D warnings`. +4. Run `cargo test --workspace`. + +Keep language changes small and include parser and SQL snapshot-style tests. User-facing syntax changes must update `docs/reference/mvp-language.md`. + +## Pull requests + +Describe the syntax or compiler behaviour changed, include representative RockQL input and generated SQL, and avoid mixing unrelated refactors with language changes. diff --git a/Cargo.toml b/Cargo.toml new file mode 100644 index 0000000..f6e86ad --- /dev/null +++ b/Cargo.toml @@ -0,0 +1,16 @@ +[workspace] +members = [ + "compiler/rockql-ast", + "compiler/rockql-parser", + "compiler/rockql-sql", + "compiler/rockql-formatter", + "compiler/rockql-cli", +] +resolver = "2" + +[workspace.package] +version = "0.1.0" +edition = "2021" +license = "Apache-2.0" +repository = "https://github.com/Sayanthrock-Developer/ROCKQL" +rust-version = "1.76" diff --git a/README.md b/README.md index 76573f1..3d0178d 100644 --- a/README.md +++ b/README.md @@ -1 +1,124 @@ -# ROCKQL \ No newline at end of file +# RockQL + +**Readable data pipelines for every database.** + +RockQL is an open-source query language and compiler that turns readable, top-to-bottom data pipelines into standard SQL. + +> **Project status:** early v0.1 compiler foundation. The syntax is experimental and may change before 1.0. + +```rockql +from employees +filter salary > 50_000 +derive yearly_salary = salary * 12 +sort {-yearly_salary} +take 10 +``` + +```sql +SELECT + *, + salary * 12 AS yearly_salary +FROM employees +WHERE salary > 50000 +ORDER BY yearly_salary DESC +LIMIT 10; +``` + +## Why RockQL? + +- Read complex queries in execution order. +- Generate SQL without replacing existing databases. +- Target multiple SQL dialects from one language. +- Receive compiler errors with exact line and column positions. +- Keep a future visual pipeline and written code synchronised. +- Run local data privately in future RockQL Studio applications. + +**Tagline:** *Write the flow. Generate the SQL.* + +## Current MVP + +The repository currently implements: + +- Rust workspace and compiler architecture. +- AST and parser for `from`, `filter`, `select`, `derive`, `sort`, and `take`. +- Newline and pipe-separated query syntax. +- Generic SQL, SQLite, and PostgreSQL targets. +- Expression normalisation for common RockQL operators and literals. +- Canonical formatter. +- CLI commands: `compile`, `check`, `format`, and `ast`. +- Parser and SQL generation tests. +- Eight GitHub Actions entry points for CI, Rust tests, web, Android, WebAssembly, releases, security, and documentation. + +## CLI + +```bash +cargo run -p rockql-cli -- compile examples/employees.rockql --target postgres +cargo run -p rockql-cli -- check examples/employees.rockql +cargo run -p rockql-cli -- format examples/employees.rockql +cargo run -p rockql-cli -- ast examples/employees.rockql +``` + +Standard input is supported: + +```bash +echo "from users | filter active == true | take 10" \ + | cargo run -q -p rockql-cli -- compile --target sqlite +``` + +Expected output: + +```sql +SELECT + * +FROM users +WHERE active = TRUE +LIMIT 10; +``` + +## Architecture + +```text +RockQL source + ↓ +Segment scanner + ↓ +Parser + ↓ +Abstract syntax tree + ↓ +SQL generator + ↓ +Formatted SQL +``` + +```text +compiler/ +├── rockql-ast/ +├── rockql-parser/ +├── rockql-sql/ +├── rockql-formatter/ +└── rockql-cli/ +``` + +The resolver, relational IR, optimiser, WebAssembly binding, playground, language server, visual studio, Android app, and additional SQL targets will be added incrementally after the compiler core is stable. + +## Roadmap + +- **v0.1:** lexer/parser foundation, core transforms, SQLite, CLI, tests. +- **v0.2:** WebAssembly compiler and browser playground. +- **v0.3:** joins, grouping, aggregation, variables, functions, PostgreSQL and MySQL depth. +- **v0.4:** formatter maturity, language server, VS Code extension, Tree-sitter grammar. +- **v0.5:** visual pipeline, schema browser, DuckDB execution, local files and history. +- **v1.0:** stable specification, compatibility policy, signed releases, Android app, JavaScript and Python bindings. + +## Project boundaries + +Early RockQL versions focus on querying and transforming data. Database administration, migrations, writes (`INSERT`, `UPDATE`, `DELETE`), cloud credential storage, team collaboration, and paid AI services are intentionally out of scope. + +## Relationship to PRQL + +PRQL is an important prior project in the pipeline-query-language space. RockQL is being developed as a distinct implementation with its own name, branding, syntax decisions, product direction, and source code. Any future reuse of third-party source must preserve the original licence, copyright notices, and attribution requirements. + +## Licence + +Licensed under the Apache License 2.0. See [LICENSE](LICENSE). diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 0000000..19be0e6 --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,5 @@ +# Security policy + +Please do not report security vulnerabilities in a public issue. Use GitHub private vulnerability reporting when it is enabled for this repository. + +RockQL v0.x is experimental. Never pass untrusted generated SQL directly to a privileged production database. Review generated SQL and use least-privilege database credentials. diff --git a/compiler/rockql-ast/Cargo.toml b/compiler/rockql-ast/Cargo.toml new file mode 100644 index 0000000..eb63dde --- /dev/null +++ b/compiler/rockql-ast/Cargo.toml @@ -0,0 +1,7 @@ +[package] +name = "rockql-ast" +version.workspace = true +edition.workspace = true +license.workspace = true +repository.workspace = true +rust-version.workspace = true diff --git a/compiler/rockql-ast/src/lib.rs b/compiler/rockql-ast/src/lib.rs new file mode 100644 index 0000000..9e32137 --- /dev/null +++ b/compiler/rockql-ast/src/lib.rs @@ -0,0 +1,69 @@ +//! Abstract syntax tree for the RockQL language. + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct Span { + pub line: usize, + pub column: usize, +} + +impl Span { + #[must_use] + pub const fn new(line: usize, column: usize) -> Self { + Self { line, column } + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Query { + pub source: Source, + pub transforms: Vec, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Source { + pub name: String, + pub span: Span, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Expr { + pub text: String, + pub span: Span, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum Transform { + Filter { + expression: Expr, + span: Span, + }, + Select { + columns: Vec, + span: Span, + }, + Derive { + name: String, + expression: Expr, + span: Span, + }, + Sort { + items: Vec, + span: Span, + }, + Take { + count: u64, + span: Span, + }, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct SortItem { + pub expression: Expr, + pub direction: SortDirection, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum SortDirection { + Ascending, + Descending, +} diff --git a/compiler/rockql-cli/Cargo.toml b/compiler/rockql-cli/Cargo.toml new file mode 100644 index 0000000..d1d43f5 --- /dev/null +++ b/compiler/rockql-cli/Cargo.toml @@ -0,0 +1,16 @@ +[package] +name = "rockql-cli" +version.workspace = true +edition.workspace = true +license.workspace = true +repository.workspace = true +rust-version.workspace = true + +[[bin]] +name = "rockql" +path = "src/main.rs" + +[dependencies] +rockql-formatter = { path = "../rockql-formatter" } +rockql-parser = { path = "../rockql-parser" } +rockql-sql = { path = "../rockql-sql" } diff --git a/compiler/rockql-cli/src/main.rs b/compiler/rockql-cli/src/main.rs new file mode 100644 index 0000000..e10de23 --- /dev/null +++ b/compiler/rockql-cli/src/main.rs @@ -0,0 +1,119 @@ +use std::env; +use std::fs; +use std::io::{self, Read}; +use std::process::ExitCode; + +use rockql_parser::parse; +use rockql_sql::{compile, Dialect}; + +fn main() -> ExitCode { + match run() { + Ok(()) => ExitCode::SUCCESS, + Err(error) => { + eprintln!("error: {error}"); + ExitCode::from(2) + } + } +} + +fn run() -> Result<(), String> { + let mut arguments = env::args().skip(1); + let command = arguments.next().unwrap_or_else(|| "help".to_owned()); + let remaining: Vec = arguments.collect(); + + match command.as_str() { + "compile" => { + let options = Options::parse(&remaining, true)?; + let source = read_source(options.input.as_deref())?; + let query = parse(&source).map_err(|error| error.to_string())?; + let sql = compile(&query, options.dialect).map_err(|error| error.to_string())?; + println!("{sql}"); + } + "check" => { + let options = Options::parse(&remaining, false)?; + let source = read_source(options.input.as_deref())?; + parse(&source).map_err(|error| error.to_string())?; + println!("RockQL query is valid."); + } + "format" => { + let options = Options::parse(&remaining, false)?; + let source = read_source(options.input.as_deref())?; + let query = parse(&source).map_err(|error| error.to_string())?; + print!("{}", rockql_formatter::format(&query)); + } + "ast" => { + let options = Options::parse(&remaining, false)?; + let source = read_source(options.input.as_deref())?; + let query = parse(&source).map_err(|error| error.to_string())?; + println!("{query:#?}"); + } + "help" | "--help" | "-h" => print_help(), + "--version" | "-V" => println!("rockql {}", env!("CARGO_PKG_VERSION")), + other => { + return Err(format!( + "unknown command `{other}`\n\nRun `rockql help` for usage." + )) + } + } + + Ok(()) +} + +#[derive(Debug)] +struct Options { + input: Option, + dialect: Dialect, +} + +impl Options { + fn parse(arguments: &[String], allow_target: bool) -> Result { + let mut input = None; + let mut dialect = Dialect::Generic; + let mut index = 0; + while index < arguments.len() { + match arguments[index].as_str() { + "--target" if allow_target => { + index += 1; + let value = arguments + .get(index) + .ok_or_else(|| "missing value after `--target`".to_owned())?; + dialect = Dialect::from_name(value).map_err(|error| error.to_string())?; + } + "-" => input = Some("-".to_owned()), + argument if argument.starts_with('-') => { + return Err(format!("unknown option `{argument}`")); + } + path => { + if input.replace(path.to_owned()).is_some() { + return Err("only one input file may be supplied".to_owned()); + } + } + } + index += 1; + } + Ok(Self { input, dialect }) + } +} + +fn read_source(path: Option<&str>) -> Result { + match path { + Some(path) if path != "-" => { + fs::read_to_string(path).map_err(|error| format!("cannot read `{path}`: {error}")) + } + _ => { + let mut source = String::new(); + io::stdin() + .read_to_string(&mut source) + .map_err(|error| format!("cannot read stdin: {error}"))?; + Ok(source) + } + } +} + +fn print_help() { + println!( + "RockQL — readable data pipelines for every database\n\n\ +Usage:\n rockql compile [FILE|-] [--target generic|sqlite|postgres]\n rockql check [FILE|-]\n rockql format [FILE|-]\n rockql ast [FILE|-]\n rockql --version\n\n\ +When FILE is omitted or `-`, RockQL reads from standard input." + ); +} diff --git a/compiler/rockql-formatter/Cargo.toml b/compiler/rockql-formatter/Cargo.toml new file mode 100644 index 0000000..8437bef --- /dev/null +++ b/compiler/rockql-formatter/Cargo.toml @@ -0,0 +1,10 @@ +[package] +name = "rockql-formatter" +version.workspace = true +edition.workspace = true +license.workspace = true +repository.workspace = true +rust-version.workspace = true + +[dependencies] +rockql-ast = { path = "../rockql-ast" } diff --git a/compiler/rockql-formatter/src/lib.rs b/compiler/rockql-formatter/src/lib.rs new file mode 100644 index 0000000..75adc1f --- /dev/null +++ b/compiler/rockql-formatter/src/lib.rs @@ -0,0 +1,59 @@ +//! Canonical formatting for RockQL source. + +use rockql_ast::{Query, SortDirection, Transform}; + +#[must_use] +pub fn format(query: &Query) -> String { + let mut output = format!("from {}", query.source.name); + for transform in &query.transforms { + output.push('\n'); + match transform { + Transform::Filter { expression, .. } => { + output.push_str("filter "); + output.push_str(expression.text.trim()); + } + Transform::Select { columns, .. } => { + output.push_str("select {"); + output.push_str( + &columns + .iter() + .map(|column| column.text.trim()) + .collect::>() + .join(", "), + ); + output.push('}'); + } + Transform::Derive { + name, expression, .. + } => { + output.push_str("derive "); + output.push_str(name); + output.push_str(" = "); + output.push_str(expression.text.trim()); + } + Transform::Sort { items, .. } => { + output.push_str("sort {"); + output.push_str( + &items + .iter() + .map(|item| { + let prefix = match item.direction { + SortDirection::Ascending => "+", + SortDirection::Descending => "-", + }; + format!("{prefix}{}", item.expression.text.trim()) + }) + .collect::>() + .join(", "), + ); + output.push('}'); + } + Transform::Take { count, .. } => { + output.push_str("take "); + output.push_str(&count.to_string()); + } + } + } + output.push('\n'); + output +} diff --git a/compiler/rockql-parser/Cargo.toml b/compiler/rockql-parser/Cargo.toml new file mode 100644 index 0000000..02ca31f --- /dev/null +++ b/compiler/rockql-parser/Cargo.toml @@ -0,0 +1,10 @@ +[package] +name = "rockql-parser" +version.workspace = true +edition.workspace = true +license.workspace = true +repository.workspace = true +rust-version.workspace = true + +[dependencies] +rockql-ast = { path = "../rockql-ast" } diff --git a/compiler/rockql-parser/src/lib.rs b/compiler/rockql-parser/src/lib.rs new file mode 100644 index 0000000..511cff1 --- /dev/null +++ b/compiler/rockql-parser/src/lib.rs @@ -0,0 +1,409 @@ +//! Parser for the RockQL MVP syntax. + +use std::error::Error; +use std::fmt::{self, Display, Formatter}; + +use rockql_ast::{Expr, Query, SortDirection, SortItem, Source, Span, Transform}; + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ParseError { + pub message: String, + pub span: Span, + pub hint: Option, +} + +impl ParseError { + fn new(message: impl Into, span: Span) -> Self { + Self { + message: message.into(), + span, + hint: None, + } + } + + fn with_hint(mut self, hint: impl Into) -> Self { + self.hint = Some(hint.into()); + self + } +} + +impl Display for ParseError { + fn fmt(&self, formatter: &mut Formatter<'_>) -> fmt::Result { + write!( + formatter, + "{} at line {}, column {}", + self.message, self.span.line, self.span.column + )?; + if let Some(hint) = &self.hint { + write!(formatter, "\nhelp: {hint}")?; + } + Ok(()) + } +} + +impl Error for ParseError {} + +#[derive(Debug, Clone)] +struct Segment { + text: String, + span: Span, +} + +/// Parse RockQL source into an AST. +/// +/// Both newline-separated pipelines and `|`-separated pipelines are accepted. +pub fn parse(source: &str) -> Result { + let segments = split_segments(source); + let first = segments.first().ok_or_else(|| { + ParseError::new("query is empty", Span::new(1, 1)) + .with_hint("start a query with `from `") + })?; + + let source_name = payload(first, "from").ok_or_else(|| { + ParseError::new("a query must start with `from`", first.span) + .with_hint("example: `from employees`") + })?; + validate_identifier_path(source_name, first.span, "source")?; + + let mut transforms = Vec::new(); + for segment in segments.iter().skip(1) { + transforms.push(parse_transform(segment)?); + } + + Ok(Query { + source: Source { + name: source_name.to_owned(), + span: first.span, + }, + transforms, + }) +} + +fn parse_transform(segment: &Segment) -> Result { + if payload(segment, "from").is_some() { + return Err(ParseError::new( + "`from` may only appear at the start of a query", + segment.span, + )); + } + + if let Some(text) = payload(segment, "filter") { + require_non_empty(text, segment.span, "filter expression")?; + return Ok(Transform::Filter { + expression: Expr { + text: text.to_owned(), + span: segment.span, + }, + span: segment.span, + }); + } + + if let Some(text) = payload(segment, "select") { + let columns = parse_expression_list(text, segment.span, "select")?; + return Ok(Transform::Select { + columns, + span: segment.span, + }); + } + + if let Some(text) = payload(segment, "derive") { + let (name, expression) = split_assignment(text).ok_or_else(|| { + ParseError::new("invalid `derive` transformation", segment.span) + .with_hint("use `derive column_name = expression`") + })?; + validate_identifier(name, segment.span, "derived column")?; + require_non_empty(expression, segment.span, "derive expression")?; + return Ok(Transform::Derive { + name: name.to_owned(), + expression: Expr { + text: expression.to_owned(), + span: segment.span, + }, + span: segment.span, + }); + } + + if let Some(text) = payload(segment, "sort") { + let body = strip_optional_braces(text, segment.span, "sort")?; + let mut items = Vec::new(); + for item in split_commas(body) { + let item = item.trim(); + require_non_empty(item, segment.span, "sort expression")?; + let (direction, expression) = match item.as_bytes().first() { + Some(b'-') => (SortDirection::Descending, item[1..].trim()), + Some(b'+') => (SortDirection::Ascending, item[1..].trim()), + _ => (SortDirection::Ascending, item), + }; + require_non_empty(expression, segment.span, "sort expression")?; + items.push(SortItem { + expression: Expr { + text: expression.to_owned(), + span: segment.span, + }, + direction, + }); + } + return Ok(Transform::Sort { + items, + span: segment.span, + }); + } + + if let Some(text) = payload(segment, "take") { + let normalized = text.replace('_', ""); + let count = normalized.parse::().map_err(|_| { + ParseError::new("`take` requires a non-negative integer", segment.span) + .with_hint("example: `take 10`") + })?; + return Ok(Transform::Take { + count, + span: segment.span, + }); + } + + let keyword = segment.text.split_whitespace().next().unwrap_or_default(); + Err( + ParseError::new(format!("unknown transformation `{keyword}`"), segment.span) + .with_hint("MVP transformations: filter, select, derive, sort, take"), + ) +} + +fn parse_expression_list(text: &str, span: Span, transform: &str) -> Result, ParseError> { + let body = strip_optional_braces(text, span, transform)?; + let mut expressions = Vec::new(); + for item in split_commas(body) { + let item = item.trim(); + require_non_empty(item, span, "expression")?; + expressions.push(Expr { + text: item.to_owned(), + span, + }); + } + if expressions.is_empty() { + return Err(ParseError::new( + format!("`{transform}` requires at least one expression"), + span, + )); + } + Ok(expressions) +} + +fn payload<'a>(segment: &'a Segment, keyword: &str) -> Option<&'a str> { + let text = segment.text.trim(); + if text == keyword { + return Some(""); + } + let rest = text.strip_prefix(keyword)?; + if rest.chars().next().is_some_and(char::is_whitespace) { + Some(rest.trim()) + } else { + None + } +} + +fn require_non_empty(text: &str, span: Span, label: &str) -> Result<(), ParseError> { + if text.trim().is_empty() { + Err(ParseError::new(format!("missing {label}"), span)) + } else { + Ok(()) + } +} + +fn strip_optional_braces<'a>( + text: &'a str, + span: Span, + transform: &str, +) -> Result<&'a str, ParseError> { + let text = text.trim(); + match (text.starts_with('{'), text.ends_with('}')) { + (true, true) => Ok(text[1..text.len() - 1].trim()), + (false, false) => Ok(text), + _ => Err(ParseError::new( + format!("unbalanced braces in `{transform}`"), + span, + )), + } +} + +fn split_assignment(text: &str) -> Option<(&str, &str)> { + let bytes = text.as_bytes(); + let mut quote = None; + let mut escaped = false; + for (index, character) in text.char_indices() { + if escaped { + escaped = false; + continue; + } + if character == '\\' && quote.is_some() { + escaped = true; + continue; + } + if matches!(character, '\'' | '"') { + if quote == Some(character) { + quote = None; + } else if quote.is_none() { + quote = Some(character); + } + continue; + } + if character == '=' && quote.is_none() { + let previous = index + .checked_sub(1) + .and_then(|value| bytes.get(value)) + .copied(); + let next = bytes.get(index + 1).copied(); + if previous.is_some_and(|value| matches!(value, b'=' | b'!' | b'<' | b'>')) + || next == Some(b'=') + { + continue; + } + return Some((text[..index].trim(), text[index + 1..].trim())); + } + } + None +} + +fn split_commas(text: &str) -> Vec<&str> { + let mut values = Vec::new(); + let mut start = 0; + let mut quote = None; + let mut depth = 0_u32; + let mut escaped = false; + for (index, character) in text.char_indices() { + if escaped { + escaped = false; + continue; + } + if character == '\\' && quote.is_some() { + escaped = true; + continue; + } + if matches!(character, '\'' | '"') { + if quote == Some(character) { + quote = None; + } else if quote.is_none() { + quote = Some(character); + } + continue; + } + if quote.is_some() { + continue; + } + match character { + '(' => depth += 1, + ')' => depth = depth.saturating_sub(1), + ',' if depth == 0 => { + values.push(&text[start..index]); + start = index + 1; + } + _ => {} + } + } + values.push(&text[start..]); + values +} + +fn validate_identifier_path(text: &str, span: Span, label: &str) -> Result<(), ParseError> { + if text.split('.').all(is_identifier) { + Ok(()) + } else { + Err(ParseError::new(format!("invalid {label} `{text}`"), span).with_hint( + "identifiers must start with a letter or underscore and contain only letters, numbers, or underscores", + )) + } +} + +fn validate_identifier(text: &str, span: Span, label: &str) -> Result<(), ParseError> { + if is_identifier(text) { + Ok(()) + } else { + Err(ParseError::new(format!("invalid {label} `{text}`"), span)) + } +} + +fn is_identifier(text: &str) -> bool { + let mut characters = text.chars(); + matches!(characters.next(), Some(first) if first == '_' || first.is_ascii_alphabetic()) + && characters.all(|character| character == '_' || character.is_ascii_alphanumeric()) +} + +fn split_segments(source: &str) -> Vec { + let mut segments = Vec::new(); + for (line_index, raw_line) in source.lines().enumerate() { + let line = raw_line.trim_end_matches('\r'); + let mut start = 0; + let mut quote = None; + let mut escaped = false; + for (index, character) in line.char_indices() { + if escaped { + escaped = false; + continue; + } + if character == '\\' && quote.is_some() { + escaped = true; + continue; + } + if matches!(character, '\'' | '"') { + if quote == Some(character) { + quote = None; + } else if quote.is_none() { + quote = Some(character); + } + continue; + } + if character == '|' && quote.is_none() { + push_segment( + &mut segments, + &line[start..index], + line_index + 1, + start + 1, + ); + start = index + 1; + } + } + push_segment(&mut segments, &line[start..], line_index + 1, start + 1); + } + segments +} + +fn push_segment(segments: &mut Vec, text: &str, line: usize, base_column: usize) { + let trimmed_start = text.trim_start(); + if trimmed_start.is_empty() || trimmed_start.starts_with('#') { + return; + } + let leading = text.len() - trimmed_start.len(); + segments.push(Segment { + text: trimmed_start.trim_end().to_owned(), + span: Span::new(line, base_column + leading), + }); +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn parses_multiline_pipeline() { + let query = parse( + "from employees\nfilter salary > 50_000\nderive yearly_salary = salary * 12\nsort {-yearly_salary}\ntake 10", + ) + .expect("query should parse"); + + assert_eq!(query.source.name, "employees"); + assert_eq!(query.transforms.len(), 4); + } + + #[test] + fn parses_pipe_pipeline() { + let query = + parse("from users | filter active == true | take 10").expect("query should parse"); + assert_eq!(query.transforms.len(), 2); + } + + #[test] + fn reports_unknown_transformation_position() { + let error = parse("from users\nexplode profile").expect_err("query should fail"); + assert_eq!(error.span, Span::new(2, 1)); + assert!(error.message.contains("unknown transformation")); + } +} diff --git a/compiler/rockql-sql/Cargo.toml b/compiler/rockql-sql/Cargo.toml new file mode 100644 index 0000000..1a77cc1 --- /dev/null +++ b/compiler/rockql-sql/Cargo.toml @@ -0,0 +1,14 @@ +[package] +name = "rockql-sql" +version.workspace = true +edition.workspace = true +license.workspace = true +repository.workspace = true +rust-version.workspace = true + +[dependencies] +rockql-ast = { path = "../rockql-ast" } + + +[dev-dependencies] +rockql-parser = { path = "../rockql-parser" } diff --git a/compiler/rockql-sql/src/lib.rs b/compiler/rockql-sql/src/lib.rs new file mode 100644 index 0000000..0c75c9a --- /dev/null +++ b/compiler/rockql-sql/src/lib.rs @@ -0,0 +1,271 @@ +//! SQL generation for RockQL queries. + +use std::error::Error; +use std::fmt::{self, Display, Formatter}; + +use rockql_ast::{Query, SortDirection, Transform}; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Dialect { + Generic, + Sqlite, + Postgres, +} + +impl Dialect { + pub fn from_name(value: &str) -> Result { + match value.to_ascii_lowercase().as_str() { + "generic" | "sql" => Ok(Self::Generic), + "sqlite" => Ok(Self::Sqlite), + "postgres" | "postgresql" => Ok(Self::Postgres), + _ => Err(SqlError(format!( + "unsupported SQL target `{value}`; expected generic, sqlite, or postgres" + ))), + } + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct SqlError(pub String); + +impl Display for SqlError { + fn fmt(&self, formatter: &mut Formatter<'_>) -> fmt::Result { + formatter.write_str(&self.0) + } +} + +impl Error for SqlError {} + +/// Compile a RockQL AST into formatted SQL. +pub fn compile(query: &Query, dialect: Dialect) -> Result { + if query.source.name.trim().is_empty() { + return Err(SqlError("query source cannot be empty".to_owned())); + } + + let mut projections = vec!["*".to_owned()]; + let mut filters = Vec::new(); + let mut sorting = Vec::new(); + let mut limit = None; + + for transform in &query.transforms { + match transform { + Transform::Filter { expression, .. } => { + filters.push(normalize_expression(&expression.text, dialect)); + } + Transform::Select { columns, .. } => { + projections = columns + .iter() + .map(|column| normalize_expression(&column.text, dialect)) + .collect(); + } + Transform::Derive { + name, expression, .. + } => { + let expression = normalize_expression(&expression.text, dialect); + projections.push(format!("{expression} AS {name}")); + } + Transform::Sort { items, .. } => { + sorting = items + .iter() + .map(|item| { + let expression = normalize_expression(&item.expression.text, dialect); + let direction = match item.direction { + SortDirection::Ascending => "ASC", + SortDirection::Descending => "DESC", + }; + format!("{expression} {direction}") + }) + .collect(); + } + Transform::Take { count, .. } => limit = Some(*count), + } + } + + let mut sql = String::from("SELECT\n"); + for (index, projection) in projections.iter().enumerate() { + sql.push_str(" "); + sql.push_str(projection); + if index + 1 != projections.len() { + sql.push(','); + } + sql.push('\n'); + } + sql.push_str("FROM "); + sql.push_str(&query.source.name); + + if !filters.is_empty() { + sql.push_str("\nWHERE "); + for (index, filter) in filters.iter().enumerate() { + if index > 0 { + sql.push_str("\n AND "); + } + sql.push_str(filter); + } + } + + if !sorting.is_empty() { + sql.push_str("\nORDER BY "); + sql.push_str(&sorting.join(", ")); + } + + if let Some(count) = limit { + sql.push_str("\nLIMIT "); + sql.push_str(&count.to_string()); + } + + sql.push(';'); + Ok(sql) +} + +fn normalize_expression(expression: &str, dialect: Dialect) -> String { + if let Some((left, right)) = split_top_level_coalesce(expression) { + return format!( + "COALESCE({}, {})", + normalize_expression(left, dialect), + normalize_expression(right, dialect) + ); + } + + let mut output = String::with_capacity(expression.len()); + let characters: Vec = expression.chars().collect(); + let mut index = 0; + let mut quote = None; + + while index < characters.len() { + let character = characters[index]; + if let Some(active_quote) = quote { + output.push(character); + if character == active_quote { + quote = None; + } else if character == '\\' && index + 1 < characters.len() { + index += 1; + output.push(characters[index]); + } + index += 1; + continue; + } + + if matches!(character, '\'' | '"') { + quote = Some(character); + output.push(character); + index += 1; + continue; + } + + if character == '=' && characters.get(index + 1) == Some(&'=') { + output.push('='); + index += 2; + continue; + } + + if character.is_ascii_digit() { + while index < characters.len() + && (characters[index].is_ascii_digit() + || characters[index] == '_' + || characters[index] == '.') + { + if characters[index] != '_' { + output.push(characters[index]); + } + index += 1; + } + continue; + } + + if character == '_' || character.is_ascii_alphabetic() { + let start = index; + index += 1; + while index < characters.len() + && (characters[index] == '_' || characters[index].is_ascii_alphanumeric()) + { + index += 1; + } + let word: String = characters[start..index].iter().collect(); + let replacement = match word.to_ascii_lowercase().as_str() { + "and" => "AND".to_owned(), + "or" => "OR".to_owned(), + "not" => "NOT".to_owned(), + "null" => "NULL".to_owned(), + "true" => boolean_literal(true, dialect).to_owned(), + "false" => boolean_literal(false, dialect).to_owned(), + _ => word, + }; + output.push_str(&replacement); + continue; + } + + output.push(character); + index += 1; + } + + output +} + +fn boolean_literal(value: bool, _dialect: Dialect) -> &'static str { + if value { + "TRUE" + } else { + "FALSE" + } +} + +fn split_top_level_coalesce(expression: &str) -> Option<(&str, &str)> { + let bytes = expression.as_bytes(); + let mut index = 0; + let mut quote = None; + let mut depth = 0_u32; + while index + 1 < bytes.len() { + let byte = bytes[index]; + if let Some(active_quote) = quote { + if byte == b'\\' { + index += 2; + continue; + } + if byte == active_quote { + quote = None; + } + index += 1; + continue; + } + match byte { + b'\'' | b'"' => quote = Some(byte), + b'(' => depth += 1, + b')' => depth = depth.saturating_sub(1), + b'?' if depth == 0 && bytes[index + 1] == b'?' => { + return Some((expression[..index].trim(), expression[index + 2..].trim())); + } + _ => {} + } + index += 1; + } + None +} + +#[cfg(test)] +mod tests { + use rockql_parser::parse; + + use super::*; + + #[test] + fn compiles_core_example() { + let query = parse( + "from employees\nfilter salary > 50_000\nderive yearly_salary = salary * 12\nsort {-yearly_salary}\ntake 10", + ) + .expect("query should parse"); + let sql = compile(&query, Dialect::Postgres).expect("query should compile"); + + assert_eq!( + sql, + "SELECT\n *,\n salary * 12 AS yearly_salary\nFROM employees\nWHERE salary > 50000\nORDER BY yearly_salary DESC\nLIMIT 10;" + ); + } + + #[test] + fn normalizes_boolean_and_equality() { + let query = + parse("from users | filter active == true | take 10").expect("query should parse"); + let sql = compile(&query, Dialect::Sqlite).expect("query should compile"); + assert!(sql.contains("WHERE active = TRUE")); + } +} diff --git a/deny.toml b/deny.toml new file mode 100644 index 0000000..2ba9e09 --- /dev/null +++ b/deny.toml @@ -0,0 +1,4 @@ +[advisories] +version = 2 +yanked = "warn" +ignore = [] diff --git a/docs/reference/mvp-language.md b/docs/reference/mvp-language.md new file mode 100644 index 0000000..0630c47 --- /dev/null +++ b/docs/reference/mvp-language.md @@ -0,0 +1,67 @@ +# RockQL MVP language reference + +RockQL v0.1 accepts a source followed by a top-to-bottom sequence of transformations. Transformations may be separated by newlines or `|` characters. + +```rockql +from employees +filter salary > 50_000 +derive yearly_salary = salary * 12 +sort {-yearly_salary} +take 10 +``` + +## Transformations + +### `from` + +Selects the source relation. It must be the first statement. + +```rockql +from analytics.employees +``` + +### `filter` + +Adds a SQL `WHERE` condition. + +```rockql +filter active == true and salary >= 50_000 +``` + +### `select` + +Chooses output expressions. Braces are optional. + +```rockql +select {id, name, email} +``` + +### `derive` + +Adds a computed output column. + +```rockql +derive yearly_salary = salary * 12 +``` + +### `sort` + +Sorts output rows. Prefix an expression with `-` for descending or `+` for ascending. + +```rockql +sort {-yearly_salary, +name} +``` + +### `take` + +Limits the number of output rows. + +```rockql +take 20 +``` + +## MVP expression normalisation + +The compiler translates `==` to SQL `=`, normalises `and`, `or`, `not`, booleans and `null`, removes numeric separators, and converts top-level `??` expressions into `COALESCE(left, right)`. + +The v0.1 parser intentionally keeps expressions lightweight. A typed expression tree, functions, dates, ranges, joins, grouping and aggregation belong to later milestones. diff --git a/examples/employees.rockql b/examples/employees.rockql new file mode 100644 index 0000000..a6b4547 --- /dev/null +++ b/examples/employees.rockql @@ -0,0 +1,5 @@ +from employees +filter salary > 50_000 +derive yearly_salary = salary * 12 +sort {-yearly_salary} +take 10 diff --git a/examples/users.rockql b/examples/users.rockql new file mode 100644 index 0000000..232850b --- /dev/null +++ b/examples/users.rockql @@ -0,0 +1 @@ +from users | filter active == true | select {id, name, email} | take 10