diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..73e9ef5 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,41 @@ +name: CI + +on: + push: + branches: [main] + pull_request: + +permissions: + contents: read + +concurrency: + group: ci-${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + rust: + name: Rust checks + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Install Rust + uses: dtolnay/rust-toolchain@1.97.1 + with: + components: rustfmt, clippy + + - name: Check formatting + run: cargo fmt --all --check + + - name: Run Clippy + run: cargo clippy --workspace --all-targets --all-features -- -D warnings + + - name: Run tests + run: cargo test --workspace --all-features + + - name: Build release CLI + run: cargo build --release --package rockql-cli + + - name: Compile example + run: ./target/release/rockql compile examples/employees.rockql --target sqlite diff --git a/.github/workflows/security.yml b/.github/workflows/security.yml new file mode 100644 index 0000000..c9b2e32 --- /dev/null +++ b/.github/workflows/security.yml @@ -0,0 +1,47 @@ +name: Security + +on: + push: + branches: [main] + pull_request: + schedule: + - cron: "23 3 * * 1" + +concurrency: + group: security-${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + dependency-review: + name: Dependency review + if: github.event_name == 'pull_request' + runs-on: ubuntu-latest + permissions: + contents: read + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Review dependency changes + uses: actions/dependency-review-action@v5 + + codeql: + name: CodeQL (Rust) + runs-on: ubuntu-latest + permissions: + actions: read + contents: read + security-events: write + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Initialise CodeQL + uses: github/codeql-action/init@v4 + with: + languages: rust + build-mode: none + + - name: Analyse + uses: github/codeql-action/analyze@v4 + with: + category: /language:rust diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..021e01e --- /dev/null +++ b/.gitignore @@ -0,0 +1,12 @@ +/target +**/node_modules +**/dist +.idea +.vscode +.DS_Store +*.db +*.sqlite +*.sqlite3 +.env +.env.* +!.env.example diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..453735d --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,63 @@ +# Contributing to RockQL + +RockQL is early-stage software. Small, focused pull requests with tests are preferred over large changes that mix language design, compiler internals, and UI work. + +## Development setup + +Requirements: + +- Rust 1.97.1 or a compatible newer toolchain. +- Git. + +```bash +git clone https://github.com/Sayanthrock-Developer/ROCKQL.git +cd ROCKQL +cargo test --workspace +``` + +## Quality checks + +Run the same checks used by CI before opening a pull request: + +```bash +cargo fmt --all --check +cargo clippy --workspace --all-targets --all-features -- -D warnings +cargo test --workspace --all-features +cargo build --release --package rockql-cli +``` + +## Language changes + +A language change should include: + +1. A clear syntax example. +2. Parser and AST updates. +3. SQL output examples for every affected dialect. +4. Positive and negative tests. +5. Documentation updates. +6. A compatibility note when existing syntax changes. + +Do not silently reinterpret valid syntax. Diagnostics should identify the exact line and column whenever possible. + +## Compiler boundaries + +Keep the stages separate: + +- Parsing creates syntax, not SQL. +- Resolution validates names and types. +- Relational IR represents database operations. +- Optimisation rewrites IR, not source text. +- Dialect generators own target-specific SQL. + +The initial compiler intentionally excludes database administration, migrations, write operations, credential storage, collaboration, and paid AI services. + +## Pull requests + +Use a descriptive title and explain: + +- The problem being solved. +- The implementation approach. +- Tests performed. +- Known limitations. + +By contributing, you agree that your contribution is licensed under Apache License 2.0. \ No newline at end of file diff --git a/Cargo.toml b/Cargo.toml new file mode 100644 index 0000000..607a5f5 --- /dev/null +++ b/Cargo.toml @@ -0,0 +1,22 @@ +[workspace] +members = [ + "compiler/rockql-ast", + "compiler/rockql-parser", + "compiler/rockql-sql", + "compiler/rockql-cli", +] +resolver = "2" + +[workspace.package] +version = "0.1.0" +edition = "2021" +license = "Apache-2.0" +repository = "https://github.com/Sayanthrock-Developer/ROCKQL" +rust-version = "1.97" + +[workspace.dependencies] +anyhow = "1.0" +clap = { version = "4.5", features = ["derive"] } +serde = { version = "1.0", features = ["derive"] } +serde_json = "1.0" +thiserror = "2.0" diff --git a/README.md b/README.md index 76573f1..ce8c7c3 100644 --- a/README.md +++ b/README.md @@ -1 +1,158 @@ -# ROCKQL \ No newline at end of file +# RockQL + +> **Readable data pipelines for every database.** + +RockQL is an open-source query language and compiler for writing database queries as readable top-to-bottom pipelines and compiling them into standard SQL. + +```rockql +from employees +filter salary > 50_000 +derive yearly_salary = salary * 12 +sort {-yearly_salary} +take 10 +``` + +```sql +SELECT + *, + salary * 12 AS yearly_salary +FROM employees +WHERE salary > 50000 +ORDER BY yearly_salary DESC +LIMIT 10; +``` + +**Write the flow. Generate the SQL.** + +## Status + +RockQL is at the compiler-foundation stage. The first implementation includes: + +- A Rust workspace. +- A serialisable abstract syntax tree. +- A parser with line and column diagnostics. +- `from`, `filter`, `select`, `derive`, `sort`, and `take`. +- Generic SQL, SQLite, and PostgreSQL output. +- `compile`, `check`, `ast`, and `format` CLI commands. +- Unit tests and GitHub Actions CI. + +The language and APIs are experimental until the v1.0 compatibility policy is published. + +## Install from source + +```bash +git clone https://github.com/Sayanthrock-Developer/ROCKQL.git +cd ROCKQL +cargo install --path compiler/rockql-cli +``` + +## CLI + +Compile a file: + +```bash +rockql compile query.rockql --target postgres +``` + +Compile stdin: + +```bash +echo "from users | filter active == true | take 10" \ + | rockql compile --target sqlite +``` + +Validate syntax: + +```bash +rockql check query.rockql +``` + +Print the AST: + +```bash +rockql ast query.rockql +``` + +Format source: + +```bash +rockql format query.rockql +rockql format query.rockql --write +``` + +## Initial syntax + +```rockql +from orders +filter status == "completed" +select customer_id, amount +derive tax = amount * 0.18 +sort {-amount} +take 20 +``` + +Supported transformations: + +- `from` +- `filter` +- `select` +- `derive` +- `sort` +- `take` + +Both newline pipelines and `|`-separated pipelines are accepted. + +## Workspace + +```text +compiler/ +├── rockql-ast/ Shared syntax model +├── rockql-parser/ Source parser and diagnostics +├── rockql-sql/ SQL dialect generator +└── rockql-cli/ Cross-platform command-line interface +``` + +Planned components include the resolver, relational IR, optimiser, formatter crate, WebAssembly bindings, browser playground, Android app, desktop app, language server, Tree-sitter grammar, and VS Code extension. + +## Compiler direction + +```text +RockQL source + ↓ +Lexer / parser + ↓ +Abstract Syntax Tree + ↓ +Name and type resolver + ↓ +Relational intermediate representation + ↓ +Query optimiser + ↓ +SQL dialect generator + ↓ +Formatted SQL +``` + +## Roadmap + +- **v0.1:** compiler foundation, SQLite generator, CLI, tests. +- **v0.2:** WebAssembly compiler, Monaco playground, diagnostics, query sharing. +- **v0.3:** joins, grouping, aggregation, variables, functions, PostgreSQL and MySQL. +- **v0.4:** formatter, language server, VS Code extension, Tree-sitter grammar. +- **v0.5:** visual pipeline editor, schema browser, DuckDB, CSV/JSON/Parquet. +- **v1.0:** stable syntax, compatibility policy, benchmarks, signed releases, Android app, JavaScript and Python bindings. + +## Project boundaries + +Early RockQL versions focus on querying and transforming data. Database administration, migrations, write operations, cloud credential storage, team collaboration, paid AI services, and complex optimisation are intentionally out of scope. + +## Design direction + +RockQL interfaces use charcoal and near-black surfaces, white primary text, muted supporting text, a restrained electric-blue or violet accent, rounded surfaces, clean monospaced code areas, subtle glass layers, and strong contrast without dashboard clutter. + +## Licence + +Licensed under the [Apache License 2.0](LICENSE). + +RockQL is an independent implementation. Third-party source code must retain its original licence, copyright notices, and required attribution. \ No newline at end of file diff --git a/compiler/rockql-ast/Cargo.toml b/compiler/rockql-ast/Cargo.toml new file mode 100644 index 0000000..2542cef --- /dev/null +++ b/compiler/rockql-ast/Cargo.toml @@ -0,0 +1,10 @@ +[package] +name = "rockql-ast" +version.workspace = true +edition.workspace = true +license.workspace = true +repository.workspace = true +rust-version.workspace = true + +[dependencies] +serde.workspace = true diff --git a/compiler/rockql-ast/src/lib.rs b/compiler/rockql-ast/src/lib.rs new file mode 100644 index 0000000..52b099a --- /dev/null +++ b/compiler/rockql-ast/src/lib.rs @@ -0,0 +1,114 @@ +use serde::{Deserialize, Serialize}; +use std::fmt::{self, Display, Formatter}; + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +pub struct Span { + pub line: usize, + pub column: usize, +} + +impl Span { + pub const fn new(line: usize, column: usize) -> Self { + Self { line, column } + } +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct Query { + pub transforms: Vec, +} + +impl Query { + pub fn new(transforms: Vec) -> Self { + Self { transforms } + } +} + +impl Display for Query { + fn fmt(&self, formatter: &mut Formatter<'_>) -> fmt::Result { + for (index, transform) in self.transforms.iter().enumerate() { + if index > 0 { + writeln!(formatter)?; + } + write!(formatter, "{}", transform.node)?; + } + Ok(()) + } +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct SpannedTransform { + pub span: Span, + pub node: Transform, +} + +impl SpannedTransform { + pub const fn new(span: Span, node: Transform) -> Self { + Self { span, node } + } +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(tag = "kind", rename_all = "snake_case")] +pub enum Transform { + From { source: String }, + Filter { expression: String }, + Select { columns: Vec }, + Derive { name: String, expression: String }, + Sort { items: Vec }, + Take { count: u64 }, +} + +impl Display for Transform { + fn fmt(&self, formatter: &mut Formatter<'_>) -> fmt::Result { + match self { + Self::From { source } => write!(formatter, "from {source}"), + Self::Filter { expression } => write!(formatter, "filter {expression}"), + Self::Select { columns } => write!(formatter, "select {}", columns.join(", ")), + Self::Derive { name, expression } => { + write!(formatter, "derive {name} = {expression}") + } + Self::Sort { items } => { + let values = items + .iter() + .map(ToString::to_string) + .collect::>() + .join(", "); + write!(formatter, "sort {{{values}}}") + } + Self::Take { count } => write!(formatter, "take {count}"), + } + } +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct SortItem { + pub expression: String, + pub descending: bool, +} + +impl SortItem { + pub fn ascending(expression: impl Into) -> Self { + Self { + expression: expression.into(), + descending: false, + } + } + + pub fn descending(expression: impl Into) -> Self { + Self { + expression: expression.into(), + descending: true, + } + } +} + +impl Display for SortItem { + fn fmt(&self, formatter: &mut Formatter<'_>) -> fmt::Result { + if self.descending { + write!(formatter, "-{}", self.expression) + } else { + write!(formatter, "{}", self.expression) + } + } +} diff --git a/compiler/rockql-cli/Cargo.toml b/compiler/rockql-cli/Cargo.toml new file mode 100644 index 0000000..be6a7f4 --- /dev/null +++ b/compiler/rockql-cli/Cargo.toml @@ -0,0 +1,20 @@ +[package] +name = "rockql-cli" +version.workspace = true +edition.workspace = true +license.workspace = true +repository.workspace = true +rust-version.workspace = true +description = "Readable data pipelines compiled to SQL" + +[[bin]] +name = "rockql" +path = "src/main.rs" + +[dependencies] +anyhow.workspace = true +clap.workspace = true +rockql-ast = { path = "../rockql-ast" } +rockql-parser = { path = "../rockql-parser" } +rockql-sql = { path = "../rockql-sql" } +serde_json.workspace = true diff --git a/compiler/rockql-cli/src/main.rs b/compiler/rockql-cli/src/main.rs new file mode 100644 index 0000000..7050f25 --- /dev/null +++ b/compiler/rockql-cli/src/main.rs @@ -0,0 +1,137 @@ +use anyhow::{anyhow, bail, Context, Result}; +use clap::{Parser, Subcommand}; +use rockql_ast::Query; +use rockql_parser::{format_source, parse as parse_rockql}; +use rockql_sql::{compile, Dialect}; +use std::fs; +use std::io::{self, Read}; +use std::path::{Path, PathBuf}; + +#[derive(Debug, Parser)] +#[command( + name = "rockql", + version, + about = "Compile readable RockQL pipelines into SQL" +)] +struct Cli { + #[command(subcommand)] + command: Command, +} + +#[derive(Debug, Subcommand)] +enum Command { + /// Compile RockQL source into SQL. + Compile { + /// RockQL file. Reads stdin when omitted. + file: Option, + /// SQL target: generic, sqlite, or postgres. + #[arg(long, default_value = "generic")] + target: String, + }, + /// Validate RockQL syntax without generating SQL. + Check { + /// RockQL file. Reads stdin when omitted. + file: Option, + }, + /// Print the parsed abstract syntax tree as JSON. + Ast { + /// RockQL file. Reads stdin when omitted. + file: Option, + }, + /// Format RockQL into canonical multiline source. + Format { + /// RockQL file. Reads stdin when omitted. + file: Option, + /// Replace the input file instead of printing formatted source. + #[arg(long)] + write: bool, + }, +} + +fn main() -> Result<()> { + let cli = Cli::parse(); + + match cli.command { + Command::Compile { file, target } => { + let input = read_input(file.as_deref())?; + let query = parse_query(&input.source, &input.label)?; + let dialect = target.parse::().map_err(anyhow::Error::msg)?; + println!("{}", compile(&query, dialect)?); + } + Command::Check { file } => { + let input = read_input(file.as_deref())?; + parse_query(&input.source, &input.label)?; + println!("OK"); + } + Command::Ast { file } => { + let input = read_input(file.as_deref())?; + let query = parse_query(&input.source, &input.label)?; + println!("{}", serde_json::to_string_pretty(&query)?); + } + Command::Format { file, write } => { + let input = read_input(file.as_deref())?; + let formatted = match format_source(&input.source) { + Ok(formatted) => formatted, + Err(diagnostics) => return report_diagnostics(&input.label, diagnostics), + }; + + if write { + let Some(path) = file else { + bail!("--write requires a file path"); + }; + fs::write(&path, formatted) + .with_context(|| format!("failed to write {}", path.display()))?; + } else { + print!("{formatted}"); + } + } + } + + Ok(()) +} + +struct Input { + source: String, + label: String, +} + +fn read_input(path: Option<&Path>) -> Result { + match path { + Some(path) => { + let source = fs::read_to_string(path) + .with_context(|| format!("failed to read {}", path.display()))?; + Ok(Input { + source, + label: path.display().to_string(), + }) + } + None => { + let mut source = String::new(); + io::stdin() + .read_to_string(&mut source) + .context("failed to read RockQL source from stdin")?; + Ok(Input { + source, + label: "".to_owned(), + }) + } + } +} + +fn parse_query(source: &str, label: &str) -> Result { + match parse_rockql(source) { + Ok(query) => Ok(query), + Err(diagnostics) => report_diagnostics(label, diagnostics), + } +} + +fn report_diagnostics(label: &str, diagnostics: Vec) -> Result { + for diagnostic in diagnostics { + eprintln!( + "{label}:{}:{}: {}", + diagnostic.span.line, diagnostic.span.column, diagnostic.message + ); + } + + Err(anyhow!("RockQL validation failed")) +} diff --git a/compiler/rockql-parser/Cargo.toml b/compiler/rockql-parser/Cargo.toml new file mode 100644 index 0000000..8af2227 --- /dev/null +++ b/compiler/rockql-parser/Cargo.toml @@ -0,0 +1,11 @@ +[package] +name = "rockql-parser" +version.workspace = true +edition.workspace = true +license.workspace = true +repository.workspace = true +rust-version.workspace = true + +[dependencies] +rockql-ast = { path = "../rockql-ast" } +serde.workspace = true diff --git a/compiler/rockql-parser/src/lib.rs b/compiler/rockql-parser/src/lib.rs new file mode 100644 index 0000000..6b22432 --- /dev/null +++ b/compiler/rockql-parser/src/lib.rs @@ -0,0 +1,277 @@ +use rockql_ast::{Query, SortItem, Span, SpannedTransform, Transform}; +use serde::{Deserialize, Serialize}; +use std::fmt::{self, Display, Formatter}; + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct Diagnostic { + pub message: String, + pub span: Span, +} + +impl Diagnostic { + fn new(message: impl Into, span: Span) -> Self { + Self { + message: message.into(), + span, + } + } +} + +impl Display for Diagnostic { + fn fmt(&self, formatter: &mut Formatter<'_>) -> fmt::Result { + write!( + formatter, + "{}:{}: {}", + self.span.line, self.span.column, self.message + ) + } +} + +#[derive(Debug)] +struct Segment { + text: String, + span: Span, +} + +pub fn parse(source: &str) -> Result> { + let segments = split_segments(source); + + if segments.is_empty() { + return Err(vec![Diagnostic::new( + "expected at least one RockQL transformation", + Span::new(1, 1), + )]); + } + + let mut transforms = Vec::with_capacity(segments.len()); + let mut diagnostics = Vec::new(); + + for segment in segments { + match parse_transform(&segment.text, segment.span) { + Ok(transform) => transforms.push(SpannedTransform::new(segment.span, transform)), + Err(diagnostic) => diagnostics.push(diagnostic), + } + } + + if diagnostics.is_empty() { + Ok(Query::new(transforms)) + } else { + Err(diagnostics) + } +} + +pub fn format_source(source: &str) -> Result> { + parse(source).map(|query| format!("{query}\n")) +} + +fn split_segments(source: &str) -> Vec { + let mut segments = Vec::new(); + + for (line_index, line) in source.lines().enumerate() { + let mut start = 0; + + for (byte_index, character) in line.char_indices() { + if character == '|' { + push_segment( + &mut segments, + &line[start..byte_index], + line_index + 1, + start, + ); + start = byte_index + character.len_utf8(); + } + } + + push_segment(&mut segments, &line[start..], line_index + 1, start); + } + + segments +} + +fn push_segment(segments: &mut Vec, raw: &str, line: usize, byte_start: usize) { + let text = raw.trim(); + if text.is_empty() { + return; + } + + let leading_bytes = raw.find(text).unwrap_or(0); + segments.push(Segment { + text: text.to_owned(), + span: Span::new(line, byte_start + leading_bytes + 1), + }); +} + +fn parse_transform(text: &str, span: Span) -> Result { + let keyword_end = text + .char_indices() + .find_map(|(index, character)| character.is_whitespace().then_some(index)) + .unwrap_or(text.len()); + + let keyword = &text[..keyword_end]; + let rest = text[keyword_end..].trim(); + + match keyword { + "from" => parse_from(rest, span), + "filter" => parse_filter(rest, span), + "select" => parse_select(rest, span), + "derive" => parse_derive(rest, span), + "sort" => parse_sort(rest, span), + "take" => parse_take(rest, span), + _ => Err(Diagnostic::new( + format!("unknown transformation `{keyword}`"), + span, + )), + } +} + +fn parse_from(rest: &str, span: Span) -> Result { + require_value(rest, "expected a table or source after `from`", span)?; + + Ok(Transform::From { + source: rest.to_owned(), + }) +} + +fn parse_filter(rest: &str, span: Span) -> Result { + require_value(rest, "expected an expression after `filter`", span)?; + + Ok(Transform::Filter { + expression: rest.to_owned(), + }) +} + +fn parse_select(rest: &str, span: Span) -> Result { + require_value(rest, "expected one or more columns after `select`", span)?; + + let columns = rest + .split(',') + .map(str::trim) + .filter(|column| !column.is_empty()) + .map(str::to_owned) + .collect::>(); + + if columns.is_empty() { + return Err(Diagnostic::new( + "expected one or more columns after `select`", + span, + )); + } + + Ok(Transform::Select { columns }) +} + +fn parse_derive(rest: &str, span: Span) -> Result { + let Some((name, expression)) = rest.split_once('=') else { + return Err(Diagnostic::new("expected `derive name = expression`", span)); + }; + + let name = name.trim(); + let expression = expression.trim(); + + require_value(name, "expected a derived column name", span)?; + require_value(expression, "expected a derived column expression", span)?; + + Ok(Transform::Derive { + name: name.to_owned(), + expression: expression.to_owned(), + }) +} + +fn parse_sort(rest: &str, span: Span) -> Result { + require_value(rest, "expected one or more sort expressions", span)?; + + let inner = match (rest.strip_prefix('{'), rest.strip_suffix('}')) { + (Some(without_open), Some(_)) => &without_open[..without_open.len().saturating_sub(1)], + (None, None) => rest, + _ => return Err(Diagnostic::new("sort braces must be balanced", span)), + }; + + let items = inner + .split(',') + .map(str::trim) + .filter(|item| !item.is_empty()) + .map(|item| { + if let Some(expression) = item.strip_prefix('-') { + SortItem::descending(expression.trim()) + } else if let Some(expression) = item.strip_prefix('+') { + SortItem::ascending(expression.trim()) + } else { + SortItem::ascending(item) + } + }) + .collect::>(); + + if items.is_empty() || items.iter().any(|item| item.expression.is_empty()) { + return Err(Diagnostic::new( + "expected one or more sort expressions", + span, + )); + } + + Ok(Transform::Sort { items }) +} + +fn parse_take(rest: &str, span: Span) -> Result { + require_value(rest, "expected a row count after `take`", span)?; + + let count = rest + .replace('_', "") + .parse::() + .map_err(|_| Diagnostic::new("`take` requires a non-negative integer row count", span))?; + + Ok(Transform::Take { count }) +} + +fn require_value(value: &str, message: &str, span: Span) -> Result<(), Diagnostic> { + if value.trim().is_empty() { + Err(Diagnostic::new(message, span)) + } else { + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn parses_multiline_pipeline() { + let query = parse( + "from employees\nfilter salary > 50_000\nderive yearly_salary = salary * 12\nsort {-yearly_salary}\ntake 10", + ) + .expect("query should parse"); + + assert_eq!(query.transforms.len(), 5); + assert_eq!(query.transforms[0].span, Span::new(1, 1)); + assert!(matches!( + query.transforms[4].node, + Transform::Take { count: 10 } + )); + } + + #[test] + fn parses_pipe_separated_pipeline() { + let query = + parse("from users | filter active == true | take 10").expect("query should parse"); + + assert_eq!(query.transforms.len(), 3); + assert_eq!(query.transforms[1].span, Span::new(1, 14)); + } + + #[test] + fn reports_line_and_column() { + let diagnostics = + parse("from users\n unknown value").expect_err("unknown transformation should fail"); + + assert_eq!(diagnostics[0].span, Span::new(2, 3)); + assert!(diagnostics[0].message.contains("unknown transformation")); + } + + #[test] + fn formats_to_canonical_multiline_source() { + let formatted = + format_source("from users | sort {-created_at} | take 5").expect("query should format"); + + assert_eq!(formatted, "from users\nsort {-created_at}\ntake 5\n"); + } +} diff --git a/compiler/rockql-sql/Cargo.toml b/compiler/rockql-sql/Cargo.toml new file mode 100644 index 0000000..a9412fb --- /dev/null +++ b/compiler/rockql-sql/Cargo.toml @@ -0,0 +1,14 @@ +[package] +name = "rockql-sql" +version.workspace = true +edition.workspace = true +license.workspace = true +repository.workspace = true +rust-version.workspace = true + +[dependencies] +rockql-ast = { path = "../rockql-ast" } +thiserror.workspace = true + +[dev-dependencies] +rockql-parser = { path = "../rockql-parser" } diff --git a/compiler/rockql-sql/src/lib.rs b/compiler/rockql-sql/src/lib.rs new file mode 100644 index 0000000..ed1f931 --- /dev/null +++ b/compiler/rockql-sql/src/lib.rs @@ -0,0 +1,267 @@ +use rockql_ast::{Query, SortItem, Transform}; +use std::fmt::{self, Display, Formatter}; +use std::str::FromStr; +use thiserror::Error; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Dialect { + Generic, + Sqlite, + Postgres, +} + +impl Display for Dialect { + fn fmt(&self, formatter: &mut Formatter<'_>) -> fmt::Result { + match self { + Self::Generic => write!(formatter, "generic"), + Self::Sqlite => write!(formatter, "sqlite"), + Self::Postgres => write!(formatter, "postgres"), + } + } +} + +impl FromStr for Dialect { + type Err = String; + + fn from_str(value: &str) -> Result { + match value.to_ascii_lowercase().as_str() { + "generic" | "sql" => Ok(Self::Generic), + "sqlite" | "sqlite3" => Ok(Self::Sqlite), + "postgres" | "postgresql" => Ok(Self::Postgres), + _ => Err(format!( + "unsupported SQL target `{value}`; expected generic, sqlite, or postgres" + )), + } + } +} + +#[derive(Debug, Error, PartialEq, Eq)] +pub enum SqlError { + #[error("query must start with a `from` transformation")] + MissingFrom, + #[error( + "only one `from` transformation is currently supported (line {line}, column {column})" + )] + MultipleFrom { line: usize, column: usize }, +} + +pub fn compile(query: &Query, dialect: Dialect) -> Result { + let Some(first) = query.transforms.first() else { + return Err(SqlError::MissingFrom); + }; + + let Transform::From { source } = &first.node else { + return Err(SqlError::MissingFrom); + }; + + let mut selected_columns: Option> = None; + let mut derived_columns = Vec::new(); + let mut filters = Vec::new(); + let mut sort_items = Vec::new(); + let mut limit = None; + + for transform in query.transforms.iter().skip(1) { + match &transform.node { + Transform::From { .. } => { + return Err(SqlError::MultipleFrom { + line: transform.span.line, + column: transform.span.column, + }) + } + Transform::Filter { expression } => { + filters.push(normalize_expression(expression, dialect)); + } + Transform::Select { columns } => { + selected_columns = Some( + columns + .iter() + .map(|column| normalize_expression(column, dialect)) + .collect(), + ); + } + Transform::Derive { name, expression } => { + derived_columns.push(format!( + "{} AS {}", + normalize_expression(expression, dialect), + name + )); + } + Transform::Sort { items } => { + sort_items.extend(items.iter().map(|item| compile_sort_item(item, dialect))); + } + Transform::Take { count } => limit = Some(*count), + } + } + + let mut select_items = selected_columns.unwrap_or_else(|| vec!["*".to_owned()]); + select_items.extend(derived_columns); + + let mut sql = String::new(); + if select_items.len() == 1 { + sql.push_str("SELECT "); + sql.push_str(&select_items[0]); + sql.push('\n'); + } else { + sql.push_str("SELECT\n"); + for (index, item) in select_items.iter().enumerate() { + sql.push_str(" "); + sql.push_str(item); + if index + 1 < select_items.len() { + sql.push(','); + } + sql.push('\n'); + } + } + + sql.push_str("FROM "); + sql.push_str(source); + sql.push('\n'); + + if !filters.is_empty() { + sql.push_str("WHERE "); + sql.push_str(&filters.join("\n AND ")); + sql.push('\n'); + } + + if !sort_items.is_empty() { + sql.push_str("ORDER BY "); + sql.push_str(&sort_items.join(", ")); + sql.push('\n'); + } + + if let Some(count) = limit { + sql.push_str("LIMIT "); + sql.push_str(&count.to_string()); + sql.push('\n'); + } + + if sql.ends_with('\n') { + sql.pop(); + } + sql.push(';'); + + Ok(sql) +} + +fn compile_sort_item(item: &SortItem, dialect: Dialect) -> String { + let direction = if item.descending { "DESC" } else { "ASC" }; + format!( + "{} {direction}", + normalize_expression(&item.expression, dialect) + ) +} + +fn normalize_expression(expression: &str, _dialect: Dialect) -> String { + let without_numeric_separators = remove_numeric_separators(expression); + let operators = without_numeric_separators + .replace("!=", "<>") + .replace("==", "="); + normalize_keywords(&operators) +} + +fn remove_numeric_separators(value: &str) -> String { + let characters = value.chars().collect::>(); + let mut output = String::with_capacity(value.len()); + + for (index, character) in characters.iter().enumerate() { + let is_numeric_separator = *character == '_' + && index > 0 + && index + 1 < characters.len() + && characters[index - 1].is_ascii_digit() + && characters[index + 1].is_ascii_digit(); + + if !is_numeric_separator { + output.push(*character); + } + } + + output +} + +fn normalize_keywords(value: &str) -> String { + let mut output = String::with_capacity(value.len()); + let mut word = String::new(); + let mut quote = None; + + let flush_word = |output: &mut String, word: &mut String| { + if word.is_empty() { + return; + } + + match word.to_ascii_lowercase().as_str() { + "true" => output.push_str("TRUE"), + "false" => output.push_str("FALSE"), + "null" => output.push_str("NULL"), + "and" => output.push_str("AND"), + "or" => output.push_str("OR"), + "not" => output.push_str("NOT"), + _ => output.push_str(word), + } + word.clear(); + }; + + for character in value.chars() { + if let Some(active_quote) = quote { + output.push(character); + if character == active_quote { + quote = None; + } + continue; + } + + if character == '\'' || character == '"' { + flush_word(&mut output, &mut word); + output.push(character); + quote = Some(character); + } else if character.is_ascii_alphanumeric() || character == '_' { + word.push(character); + } else { + flush_word(&mut output, &mut word); + output.push(character); + } + } + + flush_word(&mut output, &mut word); + output +} + +#[cfg(test)] +mod tests { + use super::*; + use rockql_parser::parse; + + #[test] + fn compiles_the_readme_example() { + let query = parse( + "from employees\nfilter salary > 50_000\nderive yearly_salary = salary * 12\nsort {-yearly_salary}\ntake 10", + ) + .expect("query should parse"); + + let sql = compile(&query, Dialect::Postgres).expect("query should compile"); + + assert_eq!( + sql, + "SELECT\n *,\n salary * 12 AS yearly_salary\nFROM employees\nWHERE salary > 50000\nORDER BY yearly_salary DESC\nLIMIT 10;" + ); + } + + #[test] + fn compiles_boolean_filter_for_sqlite() { + let query = + parse("from users | filter active == true | take 10").expect("query should parse"); + + let sql = compile(&query, Dialect::Sqlite).expect("query should compile"); + + assert_eq!(sql, "SELECT *\nFROM users\nWHERE active = TRUE\nLIMIT 10;"); + } + + #[test] + fn preserves_keywords_inside_strings() { + let query = + parse("from events | filter label == \"true and false\"").expect("query should parse"); + + let sql = compile(&query, Dialect::Generic).expect("query should compile"); + + assert!(sql.contains("label = \"true and false\"")); + } +} diff --git a/examples/employees.rockql b/examples/employees.rockql new file mode 100644 index 0000000..a6b4547 --- /dev/null +++ b/examples/employees.rockql @@ -0,0 +1,5 @@ +from employees +filter salary > 50_000 +derive yearly_salary = salary * 12 +sort {-yearly_salary} +take 10 diff --git a/rust-toolchain.toml b/rust-toolchain.toml new file mode 100644 index 0000000..9946197 --- /dev/null +++ b/rust-toolchain.toml @@ -0,0 +1,4 @@ +[toolchain] +channel = "1.97.1" +components = ["clippy", "rustfmt"] +profile = "minimal"