Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Run tests of gixsql in CI with the new preprocessor #372

Draft
wants to merge 39 commits into
base: master
Choose a base branch
from
Draft
Changes from 1 commit
Commits
Show all changes
39 commits
Select commit Hold shift + click to select a range
d879fd4
Add a skeleton for SQL preprocessor
lefessan Jun 21, 2024
951de55
SQL is parsed
LilyOlivier Jun 26, 2024
f25b81b
try to copy
LilyOlivier Jun 27, 2024
7fc9933
Add CONNECT and CONNECT RESET
LilyOlivier Jun 28, 2024
e46a5b6
light integration of typeck, size seems correctly calculated
LilyOlivier Jul 10, 2024
cdad875
typoS
LilyOlivier Jul 11, 2024
9a91b6a
refactoring
LilyOlivier Jul 26, 2024
213d2d1
Start transaction
LilyOlivier Jul 29, 2024
29e1955
Merge and conflict resolve
LilyOlivier Jul 29, 2024
500b284
drom
LilyOlivier Jul 29, 2024
61c5c87
add some Cursor utilities
LilyOlivier Aug 2, 2024
ad2bf0a
fix Select Into
LilyOlivier Aug 12, 2024
5c987a9
add Prepare Into and fix typos
LilyOlivier Aug 12, 2024
45be81d
fix Insert
LilyOlivier Aug 13, 2024
ca64c9d
fix select into and simplification
LilyOlivier Aug 14, 2024
a85b842
rework error_treatment, fix cursor, add DeclareTable
LilyOlivier Aug 16, 2024
4f9eb71
fix Cursor and fix some type calculation
LilyOlivier Aug 20, 2024
6b64ff5
somes changes
LilyOlivier Aug 28, 2024
a2be3da
better copy
LilyOlivier Sep 9, 2024
917bd30
note for the one who will continue this work
LilyOlivier Sep 20, 2024
8fa6cf1
Merge branch 'master' into sql_preproc
NeoKaios Oct 3, 2024
4e38236
fix: merge build errors
NeoKaios Oct 4, 2024
ba14fe9
fix: problem with long sql stmt generation
NeoKaios Oct 7, 2024
6e81be0
docs: how to run gixsql test for superbol
NeoKaios Oct 7, 2024
93430b2
fix: multiple bugfixes and refactors
NeoKaios Oct 11, 2024
ff2a927
fix: grammar and add cbsql to cob_extension
NeoKaios Oct 14, 2024
36c117a
feat: update gixsql testsuite doc
NeoKaios Oct 14, 2024
ff34002
fix: failing test
NeoKaios Oct 16, 2024
85cd08c
feat: add sql_ast visitor, fix typo in types
NeoKaios Oct 16, 2024
9c00487
chore: refactor, and using ast visitor
NeoKaios Oct 16, 2024
b0b7364
feat: reactivate sql preproc, go to def/ref active in exec block
NeoKaios Oct 16, 2024
d9adafd
test: autopromote
NeoKaios Oct 16, 2024
3c5fc3b
Add gixsql submodule
Halbaroth Oct 17, 2024
bcc49c0
Add wrapper for the test script
Halbaroth Oct 17, 2024
c89baa7
Add nix in the GitHub workflow
Halbaroth Oct 17, 2024
d613b9f
Add the new preprocessor
Halbaroth Oct 17, 2024
72a4bff
Use a public address for gixsql submodule
Halbaroth Oct 17, 2024
e141aac
Use the Drom templates
Halbaroth Oct 17, 2024
cb8866a
Absolute path to superbol-free in the runner
Halbaroth Oct 18, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
somes changes
LilyOlivier committed Aug 28, 2024
commit 6b64ff5b7d178bc8a86d02b1fabae20ca6ebbeed
18 changes: 14 additions & 4 deletions src/lsp/sql_ast/sql_ast.ml
Original file line number Diff line number Diff line change
@@ -21,22 +21,23 @@ type cobolVarId = string with_loc [@@deriving ord]

type cobol_var =
| CobVarNotNull of cobolVarId
| CobVarCasted of cobolVarId * sql_type
| CobVarNullIndicator of cobolVarId * cobolVarId
[@@deriving ord]

type variable =
and variable =
| SqlVar of sqlVarToken
| CobolVar of cobol_var
[@@deriving ord]

type literal =
and literal =
| LiteralVar of variable
| LiteralNum of string with_loc
| LiteralStr of string with_loc
| LiteralDot of string with_loc list
[@@deriving ord]

type sql_token =
and sql_token =
| SqlInstr of string
| SqlVarToken of variable
| SqlLit of literal
@@ -66,6 +67,7 @@ and esql_instuction =
| Rollback of rb_work_or_tran option * rb_args option
| Commit of rb_work_or_tran option * bool
| Savepoint of variable
| ReleaseSavepoint of variable
| SelectInto of
{ vars : cobol_var list;
select : sql_select;
@@ -162,6 +164,7 @@ and whenever_continuation =

and update_arg =
| WhereCurrentOf of sqlVarToken
| WhereUpdate of search_condition
| UpdateSql of sql_instruction

(*SQL*)
@@ -183,6 +186,7 @@ and from_stm = table_ref list
and table_ref =
| FromLitAs of table_ref * literal
| FromLit of literal
| FromFun of sqlVarToken * literal
| FromSelect of sql_query
| Join of table_ref * join * table_ref * join_option option

@@ -286,6 +290,7 @@ module Printer = struct
Format.fprintf fmt "COMMIT %a %s" pp_some_rb_work_or_tran rb_work_or_tran
s
| Savepoint s -> Format.fprintf fmt "SAVEPOINT %a" pp_var s
| ReleaseSavepoint s -> Format.fprintf fmt "RELEASE SAVEPOINT %a" pp_var s
| SelectInto { vars; select; select_options } ->
Format.fprintf fmt "SELECT %a INTO %a %a" pp_select_lst select pp_cob_lst
vars pp_select_options_lst select_options
@@ -375,7 +380,9 @@ module Printer = struct

and pp_where_arg fmt = function
| Some (WhereCurrentOf swhere) ->
Format.fprintf fmt "WHERE CURRENT OF %s" swhere.payload
Format.fprintf fmt "WHERE CURRENT OF %s" swhere.payload
| Some (WhereUpdate e) ->
Format.fprintf fmt "WHERE %a" pp_sql_condition e
| Some (UpdateSql sql) -> pp_sql fmt sql
| None -> ()

@@ -453,6 +460,8 @@ module Printer = struct

and pp_cob_var fmt = function
| CobVarNotNull c -> Format.fprintf fmt ":%s" c.payload
| CobVarCasted (c, t) ->
Format.fprintf fmt ":%s::%a" c.payload pp_sql_type t
| CobVarNullIndicator (c, ni) ->
Format.fprintf fmt ":%s:%s" c.payload ni.payload

@@ -548,6 +557,7 @@ module Printer = struct
and pp_table_ref fmt = function
| FromLit l -> Format.fprintf fmt "%a" pp_lit l
| FromLitAs (l, a) -> Format.fprintf fmt "%a AS %a" pp_table_ref l pp_lit a
| FromFun (v, t) -> Format.fprintf fmt "%a %a" pp_sqlVarToken v pp_lit t
| FromSelect s -> Format.fprintf fmt "(%a)" pp_sql_query s
| Join (tr1, join, tr2, opt) ->
Format.fprintf fmt "%a %s JOIN %a %a" pp_table_ref tr1 (str_join join)
9 changes: 8 additions & 1 deletion src/lsp/sql_parser/grammar.mly
Original file line number Diff line number Diff line change
@@ -67,7 +67,9 @@ let cobol_var_id :=

let cobol_var :=
| c = cobol_var_id; {CobVarNotNull c}
| c = loc(COBOL_VAR); ni=loc(COBOL_VAR); {CobVarNullIndicator(c, ni)}
| c = cobol_var_id; COLON; COLON; t = sql_type_aux; {CobVarCasted (c, t)}
(*TODO: fix this, it onely work in the context of the preproc*)
| c = cobol_var_id; ni=cobol_var_id; {CobVarNullIndicator(c, ni)}

let sql_var_name :=
| s = loc(WORD); {s}
@@ -131,6 +133,8 @@ let esql_with_opt_at :=
{ExecuteIntoUsing{executed_string; opt_into_hostref_list; opt_using_hostref_list}}
| SAVEPOINT; s= variable;
{Savepoint s}
| RELEASE; SAVEPOINT; s=variable;
{ReleaseSavepoint s}
| ROLLBACK; r=option(rb_work_or_tran); a=option(rb_args);
{Rollback(r, a)}
| COMMIT; wt= option(rb_work_or_tran); RELEASE;
@@ -207,6 +211,7 @@ let execute_immediate_arg :=

let update_arg :=
| WHERE; CURRENT; OF; v=sql_var_name; {WhereCurrentOf v}
| WHERE; v=search_condition; {WhereUpdate v}
| FROM; sql=sql; {UpdateSql( [SqlInstr "FROM"] @ sql)}

let rb_work_or_tran :=
@@ -308,6 +313,7 @@ let table_ref :=
let table_ref_simpl :=
| LPAR; select= sql_query; RPAR; {FromSelect(select)}
| LPAR; t = table_ref; RPAR; {t}
| sql_var = sql_var_name; table_name = literal; {FromFun (sql_var, table_name)}
| table_name = literal; {FromLit(table_name)}

let table_ref_non_rec :=
@@ -414,6 +420,7 @@ let sql_first_token :=
| LPAR ; {SqlInstr "(" }
| RPAR; {SqlInstr ")" }
| NOT; {SqlInstr "NOT" }
| NULL; {SqlInstr "NULL" }
| STAR; {SqlInstr "*" }
| SET; {SqlInstr "SET"}
| FOR; {SqlInstr "FOR" }
7 changes: 5 additions & 2 deletions src/lsp/sql_parser/sql_parser.ml
Original file line number Diff line number Diff line change
@@ -66,7 +66,10 @@ let parse text =
|> fst |> List.rev
in
let ast = Grammar.MenhirInterpreter.loop (supplier tokens) init_checkpoint in
(* Format.fprintf Format.std_formatter "\n%a\n" Sql_ast.Printer.pp ast; *)
(* Format.fprintf Format.std_formatter "\n%a\n" Sql_ast.Printer.pp ast; *)
ast

let parseString str = Grammar.main Lexer.token str
let parseString str =
let ast = Grammar.main Lexer.token str in
(* Format.fprintf Format.std_formatter "\n%a\n" Sql_ast.Printer.pp ast; *)
ast
3 changes: 2 additions & 1 deletion src/lsp/sql_preproc/data_gestion.ml
Original file line number Diff line number Diff line change
@@ -30,7 +30,7 @@ let add_var ~map ~name ?(length = 0) ?(vartype = 0) ?(scale = 0) ?(flags = 0)
let num = ref 0

let transform_stm map (_, stm) filename =
let prefix = " " in
let prefix = " " in
let create_new_var content ?(remplace=true) () =
let new_content = if remplace then "\"" ^ Misc.replace_colon_words content ^ "\"" else content in
let size = String.length new_content - 2 in
@@ -108,6 +108,7 @@ let transform_stm map (_, stm) filename =
(ws, map) )
| Insert _
| Savepoint _
| ReleaseSavepoint _
| Delete _ ->
let ws, map =
create_new_var (Format.asprintf "%a" Sql_ast.Printer.pp_esql tokens) ()
27 changes: 19 additions & 8 deletions src/lsp/sql_preproc/generate.ml
Original file line number Diff line number Diff line change
@@ -68,10 +68,12 @@ let generate ~filename ~contents ~cobol_unit sql_statements =
| None -> None
in

(*TODO: CobVarCasted shoould act like the casted type (rn it's ignored)*)
let cob_var_opt = function
| Some var -> (
match var with
| CobVarNotNull cobolVarId -> Some cobolVarId.payload
| CobVarCasted (var, _) -> Some var.payload
| CobVarNullIndicator (var, _) -> Some var.payload )
| None -> None
in
@@ -134,6 +136,7 @@ let generate ~filename ~contents ~cobol_unit sql_statements =
match cobol_var with
| CobVarNotNull cobolVarId ->
(cobolVarId.payload, get_length cobolVarId.payload)
| CobVarCasted (var, _) -> (var.payload, get_length var.payload)
| CobVarNullIndicator (var, _) -> (var.payload, get_length var.payload)
) )
in
@@ -280,6 +283,7 @@ let generate ~filename ~contents ~cobol_unit sql_statements =
let get_name_cobol_var (cobol_var : cobol_var) =
match cobol_var with
| CobVarNotNull c -> c.payload
| CobVarCasted (c, _) -> c.payload
| CobVarNullIndicator (c, n) -> c.payload ^ n.payload
in

@@ -435,7 +439,8 @@ let generate ~filename ~contents ~cobol_unit sql_statements =

let generate_commit prefix rb_work_or_tran rb_args ?at () =
match (rb_work_or_tran, rb_args) with
| None, false ->
| None, false
| Some Work, false ->
generate_start_end_sql prefix
[ generate_GIXSQLExec prefix "\"COMMIT\" & x\"00\"" ?at () ]
| _ -> [ Generated_type.Todo { prefix } ]
@@ -445,14 +450,15 @@ let generate ~filename ~contents ~cobol_unit sql_statements =
let name =
match var with
| [ Sql_ast.SqlVarToken (CobolVar (CobVarNotNull var)) ] -> var.payload
| [ Sql_ast.SqlVarToken (CobolVar (CobVarCasted (var, _))) ] -> var.payload
| _ ->
num := !num + 1;
"SQ" ^ string_of_int !num
in
let at_name, at_size = get_at_info at in
let fun_name = "GIXSQLExecImmediate" in
let ref_value =
let prefix = prefix ^ " " in
let prefix = prefix ^ " " in
[ Generated_type.Reference { prefix; var = "SQLCA" };
Generated_type.Reference { prefix; var = at_name };
Generated_type.Value { prefix; var = string_of_int at_size };
@@ -471,8 +477,9 @@ let generate ~filename ~contents ~cobol_unit sql_statements =
cursor_declaration := cd
in

let create_from_cursor_declaration (prefix, cur, at, var_name) =
let prefix = prefix ^ " " in
let create_from_cursor_declaration (_prefix, cur, at, var_name) =
(* let prefix = prefix ^ " " in *)
let prefix = " " in
let at_name, at_size = get_at_info at in
let cur_name, cob_var_lst, var_name, _with_hold =
match cur with
@@ -506,13 +513,13 @@ let generate ~filename ~contents ~cobol_unit sql_statements =
let fun_name, cursor_declare =
match cob_var_lst with
| [] ->
let prefix = prefix ^ " " in
let prefix = " " in
( "GIXSQLCursorDeclare",
[ Generated_type.Reference { prefix; var = var_name };
Generated_type.Value { prefix; var = "0" }
] )
| _ ->
let prefix = prefix ^ " " in
let prefix = " " in
( "GIXSQLCursorDeclareParams",
[ Generated_type.Reference { prefix; var = var_name };
Generated_type.Value { prefix; var = "0" };
@@ -522,7 +529,7 @@ let generate ~filename ~contents ~cobol_unit sql_statements =
in
let adding =
let ref_value =
let prefix = prefix ^ " " in
let prefix = " " in
[ Generated_type.Reference { prefix; var = "SQLCA" };
Generated_type.Reference { prefix; var = at_name };
Generated_type.Value { prefix; var = string_of_int at_size };
@@ -588,6 +595,8 @@ let generate ~filename ~contents ~cobol_unit sql_statements =
match sql_instr with
| [ Sql_ast.SqlVarToken (CobolVar (CobVarNotNull cobolVarId)) ] ->
cobolVarId.payload
| [ Sql_ast.SqlVarToken (CobolVar (CobVarCasted (cobolVarId, _))) ] ->
cobolVarId.payload
| _ -> failwith "Not implemented in Gix"
(*These case are not implemented in GixSql's runtime *)
in
@@ -729,7 +738,7 @@ let generate ~filename ~contents ~cobol_unit sql_statements =
let prefix = prefix ^ " " in
[ Generated_type.CallStatic
{ prefix;
fun_name = "SQGIXSQLCursorOpen";
fun_name = "GIXSQLCursorOpen";
ref_value =
(let prefix = prefix ^ " " in
[ Generated_type.Reference { prefix; var = "SQLCA" };
@@ -821,6 +830,7 @@ let generate ~filename ~contents ~cobol_unit sql_statements =

generate_insert prefix ~value_list ?at ()
| Savepoint _
| ReleaseSavepoint _
| StartTransaction ->
generate_declare prefix ?at ()
| Sql sql -> (
@@ -862,6 +872,7 @@ let generate ~filename ~contents ~cobol_unit sql_statements =
| ExecuteImmediate _
| ExecuteIntoUsing _
| Savepoint _
| ReleaseSavepoint _
| Rollback _
| Commit _
| Insert _
40 changes: 31 additions & 9 deletions src/lsp/sql_preproc/generated_type.ml
Original file line number Diff line number Diff line change
@@ -106,14 +106,15 @@ module Printer = struct
and pp_gene fmt x =
match x with
| NoChange { content } -> Format.fprintf fmt "%s\n" content
| Added { content; error_treatment; with_dot } ->
| Added { content; error_treatment; with_dot } ->
let dot =
if with_dot then
"."
else
""
in
Format.fprintf fmt "%a%a%s\n" pp_trans_stm content pp_error_treatment error_treatment dot
Format.fprintf fmt "%a%a%s\n" pp_trans_stm content pp_error_treatment
error_treatment dot
| Change { old_stms; trans_stm; error_treatment; with_dot } ->
let dot =
if with_dot then
@@ -137,15 +138,15 @@ module Printer = struct

and pp_trans_stm_aux fmt x =
match x with
| Section { name } -> Format.fprintf fmt " %s" name
| Section { name } -> Format.fprintf fmt " %s." name
| Comment { content } -> Format.fprintf fmt "ADDED *%s" content
| CallStatic { prefix; fun_name; ref_value } ->
Format.fprintf fmt "%sCALL STATIC \"%s\"%a%sEND-CALL" prefix fun_name
pp_ref_value_list ref_value prefix
| Copy { prefix; file_name } ->
Format.fprintf fmt "%sCOPY %s" prefix file_name
| GotoStatement { prefix; target } ->
Format.fprintf fmt "%sGO TO %s" prefix target
Format.fprintf fmt "%sGO TO %s." prefix target
| PerformStatement { prefix; target } ->
Format.fprintf fmt "%sPERFORM %s" prefix target
| If { prefix; condition; if_stm } ->
@@ -165,6 +166,22 @@ module Printer = struct
Format.fprintf fmt " *> WARNING: %s" content
| Todo { prefix } -> Format.fprintf fmt "%sTODO" prefix


(*TODO: maybe redo this, but nicer*)
and split_line max_length line =
let rec aux acc max_length current_line =
if String.length current_line <= max_length then
List.rev (current_line :: acc)
else
let part = String.sub current_line 0 max_length in
let rest =
String.sub current_line max_length
(String.length current_line - max_length)
in
aux ((part ^ "\"\n & \"") :: acc) 59 rest (*72 (character limit) - 12 (size of prefix ' & "' ) - 1 (for the '"')*)
in
aux [] max_length line

and pp_declaration fmt = function
| Simple_var_declaration
{ prefix; var_importance; var_name; var_type; var_content } ->
@@ -175,11 +192,15 @@ module Printer = struct
in
let var_content =
match var_content with
| Some n -> n
| Some n -> "VALUE " ^ n
| None -> ""
in
Format.fprintf fmt "%s%s %s PIC %s %s." prefix var_importance var_name
var_type var_content
let line =
Printf.sprintf "%s%s %s PIC %s %s." prefix var_importance var_name
var_type var_content
in
let lines = split_line 71 line in (*72 (character limit) - 1 (for the '"')*)
List.iter (Format.fprintf fmt "%s") lines
| Field_var_declaration { prefix; var_importance; var_name; field } ->
Format.fprintf fmt "%s%s %s.%a" prefix var_importance var_name pp_field
field
@@ -197,14 +218,15 @@ module Printer = struct
match continuation with
| Continue -> Format.fprintf fmt " CONTINUE"
| Perform sqlVarToken -> Format.fprintf fmt " PERFORM %s" sqlVarToken
| Goto sqlVarToken -> Format.fprintf fmt " GO TO %s" sqlVarToken
| Goto sqlVarToken -> Format.fprintf fmt " GO TO %s." sqlVarToken
in
let print_error fmt (not_found_whenever, str) =
match not_found_whenever with
| Some continuation ->
Format.fprintf fmt "\n%s%s\n%s%a" prefix str prefix print_continuation
continuation
| None -> Format.fprintf fmt "\n%s%s\n%s%s" prefix str prefix " CONTINUE"
| None ->
Format.fprintf fmt "\n%s%s\n%s%s" prefix str prefix " CONTINUE"
in
Format.fprintf fmt "\n%sEVALUATE TRUE%a%a%a\n%sEND-EVALUATE" prefix
print_error
Loading