Skip to content

Commit

Permalink
Improve implementation of PEP 563 - Postponed evaluation of annotatio…
Browse files Browse the repository at this point in the history 
…ns. Related to issue #2356
  • Loading branch information
@PierreQuentel
PierreQuentel committed Jan 26, 2024
1 parent 342b1aa commit 1f1f794
Show file tree
Hide file tree
Showing 9 changed files with 138 additions and 46 deletions.
70 changes: 59 additions & 11 deletions www/src/ast_to_js.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,35 @@ $B.decode_position = function(pos){
return pos
}

function get_source_from_position(src, ast_obj){
var lines = src.split('\n'),
start_line = lines[ast_obj.lineno - 1]
if(ast_obj.end_lineno == ast_obj.lineno){
return start_line.substring(ast_obj.col_offset, ast_obj.end_col_offset)
}else{
var res = start_line.substr(ast_obj.col_offset),
line_num = ast_obj.lineno + 1
while(line_num < ast_obj.end_lineno){
res += lines[line_num - 1]
}
res += lines[ast_obj.end_lineno - 1].substr(0, ast_obj.end_col_offset)
return res
}
}

function get_names(ast_obj){
// get all names used in ast object
var res = new Set()
if(ast_obj instanceof $B.ast.Name){
res.add(ast_obj)
}else if(ast_obj instanceof $B.ast.Subscript){
for(var item of get_names(ast_obj.value)){
res.add(item)
}
}
return res
}

function last_scope(scopes){
var ix = scopes.length - 1
while(scopes[ix].parent){
Expand Down Expand Up @@ -826,8 +855,6 @@ $B.ast.Assert.prototype.to_js = function(scopes){
`throw _b_.AssertionError.$factory(${msg})}\n`
}

var CO_FUTURE_ANNOTATIONS = 0x1000000

function annotation_to_str(obj){
var s
if(obj instanceof $B.ast.Name){
Expand All @@ -851,7 +878,7 @@ function annotation_to_str(obj){

$B.ast.AnnAssign.prototype.to_js = function(scopes){
var postpone_annotation = scopes.symtable.table.future.features &
CO_FUTURE_ANNOTATIONS
$B.CO_FUTURE_ANNOTATIONS
var scope = last_scope(scopes)
var js = ''
if(! scope.has_annotation){
Expand Down Expand Up @@ -2570,20 +2597,40 @@ $B.ast.FunctionDef.prototype.to_js = function(scopes){

js += `${func_ref} = ${name2}\n`
if(this.returns || parsed_args.annotations){
var ann_items = []
if(this.returns){
ann_items.push(`['return', ${this.returns.to_js(scopes)}]`)
var features = scopes.symtable.table.future.features,
postponed = features & $B.CO_FUTURE_ANNOTATIONS
if(postponed){
// PEP 563
var src = scopes.src
if(src === undefined){
console.log('no src, filename', scopes)
}
}
var ann_items = []
if(parsed_args.annotations){
for(var arg_ann in parsed_args.annotations){
var value = parsed_args.annotations[arg_ann].to_js(scopes)
if(in_class){
arg_ann = mangle(scopes, class_scope, arg_ann)
}
ann_items.push(`['${arg_ann}', ${value}]`)
if(postponed){
// PEP 563
var ann_ast = parsed_args.annotations[arg_ann],
ann_str = get_source_from_position(src, ann_ast)
ann_items.push(`['${arg_ann}', '${ann_str}']`)
}else{
var value = parsed_args.annotations[arg_ann].to_js(scopes)
ann_items.push(`['${arg_ann}', ${value}]`)
}
}
}
if(this.returns){
if(postponed){
var ann_str = get_source_from_position(src, this.returns)
ann_items.push(`['return', '${ann_str}']`)
}else{
ann_items.push(`['return', ${this.returns.to_js(scopes)}]`)
}
}

js += `${func_ref}.__annotations__ = _b_.dict.$factory([${ann_items.join(', ')}])\n`
}else{
js += `${func_ref}.__annotations__ = $B.empty_dict()\n`
Expand Down Expand Up @@ -3131,7 +3178,7 @@ $B.ast.Module.prototype.to_js = function(scopes){
js += `,\n${local_name} = locals`
}
}

js += `\nvar __file__ = frame.__file__ = '${scopes.filename || "<string>"}'\n` +
`locals.__name__ = '${name}'\n` +
`locals.__doc__ = ${extract_docstring(this, scopes)}\n`
Expand Down Expand Up @@ -3897,10 +3944,10 @@ $B.ast.YieldFrom.prototype.to_js = function(scopes){
var state = {}

$B.js_from_root = function(arg){

var ast_root = arg.ast,
symtable = arg.symtable,
filename = arg.filename,
src = arg.src,
namespaces = arg.namespaces,
imported = arg.imported

Expand All @@ -3914,6 +3961,7 @@ $B.js_from_root = function(arg){
state.filename = filename
scopes.symtable = symtable
scopes.filename = filename
scopes.src = src
scopes.namespaces = namespaces
scopes.imported = imported
scopes.imports = {}
Expand Down
55 changes: 35 additions & 20 deletions www/src/brython.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -71,6 +71,7 @@ while(ix < minlen && short[ix]==long[ix]){ix++}
$B.tz_name=long.substr(ix).trim()
$B.PyCF_ONLY_AST=1024
$B.PyCF_TYPE_COMMENTS=0x1000
$B.CO_FUTURE_ANNOTATIONS=0x1000000
if($B.isWebWorker){$B.charset="utf-8"}else{
$B.charset=document.characterSet ||document.inputEncoding ||"utf-8"}
$B.max_int=Math.pow(2,53)-1
Expand Down Expand Up @@ -158,8 +159,8 @@ $B.stdlib_module_names=Object.keys($B.stdlib)})(__BRYTHON__)
;
__BRYTHON__.implementation=[3,12,1,'dev',0]
__BRYTHON__.version_info=[3,12,0,'final',0]
__BRYTHON__.compiled_date="2024-01-25 22:03:04.718113"
__BRYTHON__.timestamp=1706216584718
__BRYTHON__.compiled_date="2024-01-26 11:06:37.294410"
__BRYTHON__.timestamp=1706263597294
__BRYTHON__.builtin_module_names=["_ajax","_ast","_base64","_binascii","_io_classes","_json","_jsre","_locale","_multiprocessing","_posixsubprocess","_profile","_random","_sre","_sre_utils","_string","_strptime","_svg","_symtable","_tokenize","_webcomponent","_webworker","_zlib_utils","array","builtins","dis","encoding_cp932","hashlib","html_parser","marshal","math","modulefinder","posix","python_re","python_re_new","unicodedata"]
;
(function($B){var _b_=$B.builtins
Expand Down Expand Up @@ -739,7 +740,6 @@ if(attrs.length > 0){res+='\n'
res+=attrs.map(x=> ' '.repeat(indent+1)+x).join(',\n')}
res+=')'
return res}
var CO_FUTURE_ANNOTATIONS=0x1000000
function get_line(filename,lineno){var src=$B.file_cache[filename],line=_b_.None
if(src !==undefined){var lines=src.split('\n')
line=lines[lineno-1]}
Expand All @@ -753,7 +753,7 @@ i++}}
while(i < mod.body.length){var child=mod.body[i]
if(child instanceof $B.ast.ImportFrom && child.module=='__future__'){
for(var alias of child.names){var name=alias.name
if(name=="braces"){raise_error_known_location(_b_.SyntaxError,filename,alias.lineno,alias.col_offset,alias.end_lineno,alias.end_col_offset,get_line(filename,child.lineno),"not a chance")}else if(name=="annotations"){features |=CO_FUTURE_ANNOTATIONS}else if(VALID_FUTURES.indexOf(name)==-1){raise_error_known_location(_b_.SyntaxError,filename,alias.lineno,alias.col_offset,alias.end_lineno,alias.end_col_offset,get_line(filename,child.lineno),`future feature ${name} is not defined`)}}
if(name=="braces"){raise_error_known_location(_b_.SyntaxError,filename,alias.lineno,alias.col_offset,alias.end_lineno,alias.end_col_offset,get_line(filename,child.lineno),"not a chance")}else if(name=="annotations"){features |=$B.CO_FUTURE_ANNOTATIONS}else if(VALID_FUTURES.indexOf(name)==-1){raise_error_known_location(_b_.SyntaxError,filename,alias.lineno,alias.col_offset,alias.end_lineno,alias.end_col_offset,get_line(filename,child.lineno),`future feature ${name} is not defined`)}}
i++}else{break}}
return{features}}
function set_position(ast_obj,position,end_position){ast_obj.lineno=position.start[0]
Expand Down Expand Up @@ -4041,8 +4041,8 @@ this.parent=C
this.tree=[]}
SubscripCtx.prototype.ast=function(){var slice
if(this.tree.length > 1){var slice_items=this.tree.map(x=> x.ast())
slice=new ast.Tuple(slice_items)
set_position(slice,this.position,this.end_position)}else{slice=this.tree[0].ast()}
slice=new ast.Tuple(slice_items)}else{slice=this.tree[0].ast()}
set_position(slice,this.position,this.end_position)
slice.ctx=new ast.Load()
var value=this.value.ast()
if(value.ctx){value.ctx=new ast.Load()}
Expand Down Expand Up @@ -4853,7 +4853,7 @@ _ast=root.ast()}
$B.parse_time+=globalThis.performance.now()-t0
var future=$B.future_features(_ast,filename)
var symtable=$B._PySymtable_Build(_ast,filename,future)
var js_obj=$B.js_from_root({ast:_ast,symtable,filename,imported})
var js_obj=$B.js_from_root({ast:_ast,symtable,filename,src,imported})
var js_from_ast=js_obj.js
return{
_ast,imports:js_obj.imports,to_js:function(){return js_from_ast}}}
Expand Down Expand Up @@ -6814,7 +6814,7 @@ $._ast=$B.ast_js_to_py(_ast)
$._ast.$js_ast=_ast
var future=$B.future_features(_ast,filename)
var symtable=$B._PySymtable_Build(_ast,filename,future)
$B.js_from_root({ast:_ast,symtable,filename,})
$B.js_from_root({ast:_ast,symtable,filename,src:$.source})
return $}
_b_.debug=$B.debug > 0
_b_.delattr=function(obj,attr){
Expand Down Expand Up @@ -6925,7 +6925,7 @@ root.mode=mode
root.filename=filename
$B.parser.dispatch_tokens(root)
_ast=root.ast()}}
var future=$B.future_features(_ast,filename),symtable=$B._PySymtable_Build(_ast,filename,future),js_obj=$B.js_from_root({ast:_ast,symtable,filename,namespaces:{local_name,exec_locals,global_name,exec_globals}}),js=js_obj.js}catch(err){if(err.args){if(err.args[1]){exec_locals.$lineno=err.args[1][1]}}else{console.log('JS Error',err.message)}
var future=$B.future_features(_ast,filename),symtable=$B._PySymtable_Build(_ast,filename,future),js_obj=$B.js_from_root({ast:_ast,symtable,filename,src,namespaces:{local_name,exec_locals,global_name,exec_globals}}),js=js_obj.js}catch(err){if(err.args){if(err.args[1]){exec_locals.$lineno=err.args[1][1]}}else{console.log('JS Error',err.message)}
$B.frame_obj=save_frame_obj
throw err}
if(mode=='eval'){
Expand Down Expand Up @@ -15292,6 +15292,15 @@ target.end_lineno=origin.end_lineno
target.end_col_offset=origin.end_col_offset}
function encode_position(a,b,c,d){if(d===undefined){return `[${[a, b, c]}]`}else{return `[${[a, b, c, d]}]`}}
$B.decode_position=function(pos){return pos}
function get_source_from_position(src,ast_obj){var lines=src.split('\n'),start_line=lines[ast_obj.lineno-1]
if(ast_obj.end_lineno==ast_obj.lineno){return start_line.substring(ast_obj.col_offset,ast_obj.end_col_offset)}else{var res=start_line.substr(ast_obj.col_offset),line_num=ast_obj.lineno+1
while(line_num < ast_obj.end_lineno){res+=lines[line_num-1]}
res+=lines[ast_obj.end_lineno-1].substr(0,ast_obj.end_col_offset)
return res}}
function get_names(ast_obj){
var res=new Set()
if(ast_obj instanceof $B.ast.Name){res.add(ast_obj)}else if(ast_obj instanceof $B.ast.Subscript){for(var item of get_names(ast_obj.value)){res.add(item)}}
return res}
function last_scope(scopes){var ix=scopes.length-1
while(scopes[ix].parent){ix--}
return scopes[ix]}
Expand Down Expand Up @@ -15571,13 +15580,12 @@ if(check_func){obj._check()}}
$B.ast.Assert.prototype.to_js=function(scopes){var test=$B.js_from_ast(this.test,scopes),msg=this.msg ? $B.js_from_ast(this.msg,scopes):''
return `if($B.set_lineno(frame, ${this.lineno}) && !$B.$bool(${test})){\n`+
`throw _b_.AssertionError.$factory(${msg})}\n`}
var CO_FUTURE_ANNOTATIONS=0x1000000
function annotation_to_str(obj){var s
if(obj instanceof $B.ast.Name){s=obj.id}else if(obj instanceof $B.ast.BinOp){s=annotation_to_str(obj.left)+'|'+annotation_to_str(obj.right)}else if(obj instanceof $B.ast.Subscript){s=annotation_to_str(obj.value)+'['+
annotation_to_str(obj.slice)+']'}else if(obj instanceof $B.ast.Constant){if(obj.value===_b_.None){s='None'}else{console.log('other constant',obj)}}else{console.log('other annotation',obj)}
return s}
$B.ast.AnnAssign.prototype.to_js=function(scopes){var postpone_annotation=scopes.symtable.table.future.features &
CO_FUTURE_ANNOTATIONS
$B.CO_FUTURE_ANNOTATIONS
var scope=last_scope(scopes)
var js=''
if(! scope.has_annotation){js+='locals.__annotations__ = locals.__annotations__ || $B.empty_dict()\n'
Expand Down Expand Up @@ -16396,11 +16404,18 @@ var mangled=mangle(scopes,func_name_scope,this.name),func_ref=`${make_scope_name
if(decorated){func_ref=`decorated${$B.UUID()}`
js+='var '}
js+=`${func_ref} = ${name2}\n`
if(this.returns ||parsed_args.annotations){var ann_items=[]
if(this.returns){ann_items.push(`['return', ${this.returns.to_js(scopes)}]`)}
if(parsed_args.annotations){for(var arg_ann in parsed_args.annotations){var value=parsed_args.annotations[arg_ann].to_js(scopes)
if(in_class){arg_ann=mangle(scopes,class_scope,arg_ann)}
ann_items.push(`['${arg_ann}', ${value}]`)}}
if(this.returns ||parsed_args.annotations){var features=scopes.symtable.table.future.features,postponed=features & $B.CO_FUTURE_ANNOTATIONS
if(postponed){
var src=scopes.src
if(src===undefined){console.log('no src, filename',scopes)}}
var ann_items=[]
if(parsed_args.annotations){for(var arg_ann in parsed_args.annotations){if(in_class){arg_ann=mangle(scopes,class_scope,arg_ann)}
if(postponed){
var ann_ast=parsed_args.annotations[arg_ann],ann_str=get_source_from_position(src,ann_ast)
ann_items.push(`['${arg_ann}', '${ann_str}']`)}else{var value=parsed_args.annotations[arg_ann].to_js(scopes)
ann_items.push(`['${arg_ann}', ${value}]`)}}}
if(this.returns){if(postponed){var ann_str=get_source_from_position(src,this.returns)
ann_items.push(`['return', '${ann_str}']`)}else{ann_items.push(`['return', ${this.returns.to_js(scopes)}]`)}}
js+=`${func_ref}.__annotations__ = _b_.dict.$factory([${ann_items.join(', ')}])\n`}else{js+=`${func_ref}.__annotations__ = $B.empty_dict()\n`}
if(has_type_params){scopes.pop()}
if(decorated && ! has_type_params){js+=`${make_scope_name(scopes, func_name_scope)}.${mangled} = `
Expand Down Expand Up @@ -17023,13 +17038,14 @@ return `yield* (function* f(){
return _r${n}
})()`}
var state={}
$B.js_from_root=function(arg){var ast_root=arg.ast,symtable=arg.symtable,filename=arg.filename,namespaces=arg.namespaces,imported=arg.imported
$B.js_from_root=function(arg){var ast_root=arg.ast,symtable=arg.symtable,filename=arg.filename,src=arg.src,namespaces=arg.namespaces,imported=arg.imported
if($B.show_ast_dump){console.log($B.ast_dump(ast_root))}
if($B.compiler_check){$B.compiler_check(ast_root,symtable)}
var scopes=[]
state.filename=filename
scopes.symtable=symtable
scopes.filename=filename
scopes.src=src
scopes.namespaces=namespaces
scopes.imported=imported
scopes.imports={}
Expand Down Expand Up @@ -17068,7 +17084,6 @@ DEF_COMP_CELL=2 << 10
var DEF_BOUND=DEF_LOCAL |DEF_PARAM |DEF_IMPORT
var SCOPE_OFFSET=12,SCOPE_MASK=(DEF_GLOBAL |DEF_LOCAL |DEF_PARAM |DEF_NONLOCAL)
var LOCAL=1,GLOBAL_EXPLICIT=2,GLOBAL_IMPLICIT=3,FREE=4,CELL=5
var CO_FUTURE_ANNOTATIONS=0x1000000
var TYPE_MODULE=2
var NULL=undefined
var ModuleBlock=2,ClassBlock=1,FunctionBlock=0,AnnotationBlock=4,TypeVarBoundBlock=5,TypeAliasBlock=6,TypeParamBlock=7
Expand Down Expand Up @@ -17834,7 +17849,7 @@ visitor.params=function(st,args){if(! args){return-1}
for(var arg of args){if(! symtable_add_def(st,arg.arg,DEF_PARAM,LOCATION(arg)))
return 0}
return 1}
visitor.annotation=function(st,annotation){var future_annotations=st.future.features & CO_FUTURE_ANNOTATIONS
visitor.annotation=function(st,annotation){var future_annotations=st.future.features & $B.CO_FUTURE_ANNOTATIONS
if(future_annotations &&
!symtable_enter_block(st,'_annotation',AnnotationBlock,annotation,annotation.lineno,annotation.col_offset,annotation.end_lineno,annotation.end_col_offset)){VISIT_QUIT(st,0)}
VISIT(st,expr,annotation)
Expand All @@ -17843,7 +17858,7 @@ return 1}
visitor.argannotations=function(st,args){if(!args){return-1}
for(var arg of args){if(arg.annotation){VISIT(st,expr,arg.annotation)}}
return 1}
visitor.annotations=function(st,o,a,returns){var future_annotations=st.future.ff_features & CO_FUTURE_ANNOTATIONS;
visitor.annotations=function(st,o,a,returns){var future_annotations=st.future.ff_features & $B.CO_FUTURE_ANNOTATIONS;
if(future_annotations &&
!symtable_enter_block(st,'_annotation',AnnotationBlock,o,o.lineno,o.col_offset,o.end_lineno,o.end_col_offset)){VISIT_QUIT(st,0);}
if(a.posonlyargs && !visitor.argannotations(st,a.posonlyargs))
Expand Down
1 change: 1 addition & 0 deletions www/src/brython_builtins.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -172,6 +172,7 @@ $B.tz_name = long.substr(ix).trim()
// compiler flags, used in libs/_ast.js and compile()
$B.PyCF_ONLY_AST = 1024
$B.PyCF_TYPE_COMMENTS = 0x1000
$B.CO_FUTURE_ANNOTATIONS = 0x1000000

if($B.isWebWorker){
$B.charset = "utf-8"
Expand Down
2 changes: 1 addition & 1 deletion www/src/brython_stdlib.js
View file

Large diffs are not rendered by default.

10 changes: 4 additions & 6 deletions www/src/py2js.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -239,9 +239,6 @@ var ast_dump = $B.ast_dump = function(tree, indent){
return res
}

// Get options set by "from __future__ import XXX"
var CO_FUTURE_ANNOTATIONS = 0x1000000

function get_line(filename, lineno){
var src = $B.file_cache[filename],
line = _b_.None
Expand Down Expand Up @@ -287,7 +284,7 @@ $B.future_features = function(mod, filename){
get_line(filename, child.lineno),
"not a chance")
}else if(name == "annotations"){
features |= CO_FUTURE_ANNOTATIONS
features |= $B.CO_FUTURE_ANNOTATIONS
}else if(VALID_FUTURES.indexOf(name) == -1){
raise_error_known_location(_b_.SyntaxError, filename,
alias.lineno, alias.col_offset,
Expand Down Expand Up @@ -6965,10 +6962,10 @@ SubscripCtx.prototype.ast = function(){
if(this.tree.length > 1){
var slice_items = this.tree.map(x => x.ast())
slice = new ast.Tuple(slice_items)
set_position(slice, this.position, this.end_position)
}else{
slice = this.tree[0].ast()
}
set_position(slice, this.position, this.end_position)
slice.ctx = new ast.Load()
var value = this.value.ast()
if(value.ctx){
Expand Down Expand Up @@ -8916,6 +8913,7 @@ $B.py2js = function(src, module, locals_id, parent_scope){
var js_obj = $B.js_from_root({ast: _ast,
symtable,
filename,
src,
imported})
var js_from_ast = js_obj.js

Expand Down Expand Up @@ -9338,7 +9336,7 @@ function run_scripts(_scripts){
$B.file_cache[filename] = src
$B.url2name[filename] = module_name
$B.scripts[filename] = script
$B.tasks.push([$B.run_script, script, src, module_name,
$B.tasks.push([$B.run_script, script, src, module_name,
$B.script_path, true])
}
}
Expand Down
4 changes: 3 additions & 1 deletion www/src/py_builtin_functions.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -396,7 +396,7 @@ _b_.compile = function() {
filename, err_token.lineno, err_token.col_offset,
err_token.end_lineno, err_token.end_col_offset,
err_token.line, 'invalid syntax')
}
}
}catch(err){
if($.mode == 'single'){
try{
Expand Down Expand Up @@ -493,6 +493,7 @@ _b_.compile = function() {
ast: _ast,
symtable,
filename,
src: $.source
})

return $
Expand Down Expand Up @@ -821,6 +822,7 @@ var $$eval = _b_.eval = function(){
js_obj = $B.js_from_root({ast: _ast,
symtable,
filename,
src,
namespaces: {local_name,
exec_locals,
global_name,
Expand Down
Loading

0 comments on commit 1f1f794

Please sign in to comment.