chore: merge upstream opengrep/main

install.ps1

@@ -333,9 +333,26 @@ function Main {
             }
 
             Write-Host "Testing binary..."
-            # Test by calling --version on the downloaded binary
-            $testOutput = & $binaryPath --version 2>&1
-            if (-not $testOutput -or $LASTEXITCODE -ne 0) {
+            # Test by calling --version on the downloaded binary.
+            # We route stderr through a temp file rather than using `2>&1`, so
+            # that harmless runtime warnings (e.g. requests' RequestsDependency-
+            # Warning) are not surfaced by PowerShell as NativeCommandError
+            # records and misinterpreted as failures. We still surface them to
+            # the user via Write-Host, and rely on $LASTEXITCODE to decide
+            # whether the binary actually ran.
+            $stderrFile = New-TemporaryFile
+            try {
+                $testOutput = & $binaryPath --version 2>$stderrFile
+                $testExit   = $LASTEXITCODE
+                $testStderr = (Get-Content -Raw -ErrorAction SilentlyContinue $stderrFile)
+            }
+            finally {
+                Remove-Item -Force -ErrorAction SilentlyContinue $stderrFile
+            }
+            if ($testStderr) {
+                Write-Host $testStderr
+            }
+            if ($testExit -ne 0 -or -not $testOutput) {
                 throw "Failed to execute installed binary: $binaryPath"
             }
 

languages/elixir/ast/AST_elixir.ml

@@ -209,6 +209,10 @@ and expr =
   | DotAnon of expr * tok
   (* only inside Call *)
   | DotRemote of remote_dot
+  (* Elixir map/struct field access: `foo.bar` with no parens, no args,
+   * no do-block. Distinct from `foo.bar(...)` which is a remote call
+   * (encoded as `Call (DotRemote _, _, _)`). *)
+  | FieldAccess of remote_dot
   | ModuleVarAccess of tok (* @ *) * expr
   | ArrayAccess of expr * expr bracket
   (* a Call can be a thousand things, including function and module definitions

languages/elixir/generic/Elixir_to_generic.ml

@@ -118,6 +118,40 @@ let expr_of_expr_or_kwds (x : (G.expr, keywords_generic) Either_.t) : G.expr =
   | Left e -> e
   | Right kwds -> list_container_of_kwds kwds
 
+(* This is a modified version of Ast_generic_helpers.expr_to_pattern *)
+let rec expr_to_pattern (e : G.expr) : G.pattern =
+  match e.e with
+  | G.N (G.Id (id, info)) -> G.PatId (id, info)
+  | G.Container (G.Tuple, (t1, xs, t2)) ->
+      G.PatTuple (t1, List_.map expr_to_pattern xs, t2)
+  | G.L l -> G.PatLiteral l
+  | G.Container ((List | Dict), (t1, xs, t2)) ->
+      G.PatList (t1, List_.map expr_to_pattern xs, t2)
+  | G.Constructor (n, (_, args, _)) ->
+      G.PatConstructor (n, List_.map expr_to_pattern args)
+  | G.Ellipsis t -> G.PatEllipsis t
+  | G.OtherExpr (tag, [ G.E e ]) -> G.OtherPat (tag, [ G.P (expr_to_pattern e) ])
+  | G.Cast (ty, _tok, expr) -> G.PatTyped (expr_to_pattern expr, ty)
+  | G.LetPattern (p, {e = G.N (G.Id (i, info)); _} ) -> G.PatAs (p, (i, info))
+  | G.Call (f, args) ->
+      begin match f.e, Tok.unbracket args with
+      | G.N (G.Id (("<>", _), _) as n),
+        [ G.Arg ({ e = G.L (G.String _); _ } as l);
+          G.Arg ({ e = G.N _; _ } as r) ] ->
+          G.PatConstructor (n, [ expr_to_pattern l; expr_to_pattern r ])
+      | G.N (G.Id (("^", _), _)),
+        [ G.Arg ({ e = G.N _; _ } as rhs) ] ->
+          let tmp = "__tmp", Tok.unsafe_fake_tok "__tmp" in
+          let tmp_info = G.empty_id_info ~hidden:true () in
+          let lhs = G.N (G.Id (tmp, tmp_info)) |> G.e in
+          let op = G.IdSpecial (G.Op G.Eq, Tok.unsafe_fake_tok "==") |> G.e in
+          let cmp = G.Call (op, Tok.unsafe_fake_bracket [ G.Arg lhs; G.Arg rhs ]) |> G.e in
+          G.PatWhen (G.PatId (tmp, tmp_info), cmp)
+      | _ -> OtherPat (("ExprToPattern", Tok.unsafe_fake_tok ""), [ G.E e ])
+      end
+  (* TODO: PatKeyVal and more *)
+  | _ -> OtherPat (("ExprToPattern", Tok.unsafe_fake_tok ""), [ G.E e ])
+
 (* TODO: lots of work here to detect when args is really a single
  * pattern, or tuples *)
 let pat_of_args_and_when (args, when_opt) : G.pattern =
@@ -129,8 +163,8 @@ let pat_of_args_and_when (args, when_opt) : G.pattern =
   let pats =
     List_.map
       (function
-        | G.OtherArg (("ArgKwdQuoted", _), [ G.E e ]) -> H.expr_to_pattern e
-        | arg -> H.argument_to_expr arg |> H.expr_to_pattern)
+        | G.OtherArg (("ArgKwdQuoted", _), [ G.E e ]) -> expr_to_pattern e
+        | arg -> H.argument_to_expr arg |> expr_to_pattern)
       args
   in
   let pat =
@@ -432,7 +466,7 @@ and map_stmt env (v : stmt) : G.stmt =
       let comp_clauses = List_.map (fun (clause : for_clause) ->
         match clause with
         | ForGenerator (pat, tarrow, collection) ->
-            let pat = map_expr env pat |> H.expr_to_pattern in
+            let pat = map_expr env pat |> expr_to_pattern in
             let collection = map_expr env collection in
             G.CompFor (tfor, pat, tarrow, collection)
         | ForFilter e ->
@@ -494,12 +528,12 @@ and map_param_to_gparam env (p : parameter) : G.parameter =
           G.Param (G.param_of_id ?pdefault id))
   | OtherParamExpr e ->
       let e = map_expr env e in
-      G.ParamPattern (H.expr_to_pattern e)
+      G.ParamPattern (expr_to_pattern e)
   | OtherParamPair (kwd, e) ->
       let kwd = map_keyword env kwd in
       let e = map_expr env e in
       let e = keyval_of_pair (Left (kwd, e)) in
-      G.ParamPattern (H.expr_to_pattern e)
+      G.ParamPattern (expr_to_pattern e)
 
 (* Convert one rescue/catch stab clause to a G.catch arm.
  * Each stab has a list of exception-type expressions and a handler body. *)
@@ -510,9 +544,9 @@ and map_rescue_stab_to_catch env tok (stab : stab_clause) : G.catch =
     | [] -> G.PatEllipsis tok
     | [arg] ->
         let e = map_expr env arg in
-        H.expr_to_pattern e
+        expr_to_pattern e
     | args ->
-        let pats = List_.map (fun a -> H.expr_to_pattern (map_expr env a)) args in
+        let pats = List_.map (fun a -> expr_to_pattern (map_expr env a)) args in
         let pat =
           List.fold_right (fun p acc -> G.DisjPat (p, acc))
             (List.tl pats) (List.hd pats)
@@ -708,10 +742,10 @@ and map_vardef env v1 v2 =
 (* TODO: Elixir also has these patterns:
  *   ^x = 0 meaning x cannot be re-assigned later, and
  *   [x|y] = [0, 1, 2] where x maps to 0, and y maps to the rest
- * and H.expr_to_pattern doesn't cover these cases.
+ * and expr_to_pattern doesn't cover these cases.
  *)
 and map_letpattern env v1 v2 =
-  let e1 = H.expr_to_pattern (map_expr env v1) in
+  let e1 = expr_to_pattern (map_expr env v1) in
   let e2 = map_expr env v2 in
   G.LetPattern (e1, e2) |> G.e
 
@@ -844,6 +878,10 @@ and map_expr env v : G.expr =
       G.OtherExpr (("DotAnon", tdot), [ G.E e ]) |> G.e
   (* only inside a Call *)
   | DotRemote v -> map_remote_dot env v
+  (* Elixir field access: `foo.bar` (no parens). Translate to a plain
+   * DotAccess so downstream analysis treats it as field access, not a
+   * zero-arity function call. *)
+  | FieldAccess v -> map_remote_dot env v
   | ModuleVarAccess (tat, v2) ->
       let e = map_expr env v2 in
       G.OtherExpr (("AttrExpr", tat), [ G.E e ]) |> G.e

languages/elixir/tree-sitter/Parse_elixir_tree_sitter.ml

@@ -546,10 +546,10 @@ and map_body (env : env) ((v1, v2, v3, v4) : CST.body) : body =
   let _v4 = map_terminator_opt env v4 in
   v2 :: v3
 
-and map_call (env : env) (x : CST.call) : call =
+and map_call (env : env) (x : CST.call) : expr =
   match x with
   | `Call_with_parens_b98484c x -> map_call_without_parentheses env x
-  | `Call_with_parens_403315d x -> map_call_with_parentheses env x
+  | `Call_with_parens_403315d x -> Call (map_call_with_parentheses env x)
 
 and map_call_arguments_with_parentheses (env : env)
     ((v1, v2, v3) : CST.call_arguments_with_parentheses) : arguments bracket =
@@ -627,26 +627,32 @@ and map_call_with_parentheses (env : env) (x : CST.call_with_parentheses) : call
       mk_call_parens (Call call1) args blopt
 
 and map_call_without_parentheses (env : env) (x : CST.call_without_parentheses)
-    : call =
+    : expr =
   match x with
   | `Local_call_with_parens (v1, v2, v3) ->
       let id = map_identifier env v1 in
       let args = map_call_arguments_without_parentheses env v2 in
       let blopt = map_anon_opt_opt_nl_before_do_do_blk_3eff85f env v3 in
-      mk_call_no_parens (Left id) args blopt
+      Call (mk_call_no_parens (Left id) args blopt)
   | `Local_call_just_do_blk (v1, v2) ->
       let id = map_identifier env v1 in
       let bl = map_do_block env v2 in
-      mk_call_no_parens (Left id) ([], []) (Some bl)
+      Call (mk_call_no_parens (Left id) ([], []) (Some bl))
   | `Remote_call_with_parens (v1, v2, v3) ->
       let rdot = map_remote_dot env v1 in
-      let args : arguments =
-        match v2 with
-        | Some x -> map_call_arguments_without_parentheses env x
-        | None -> ([], [])
-      in
       let blopt = map_anon_opt_opt_nl_before_do_do_blk_3eff85f env v3 in
-      mk_call_no_parens (Right rdot) args blopt
+      (match v2, blopt with
+       | None, None ->
+           (* Elixir map/struct field access: `foo.bar` with no parens,
+            * no args, no do-block. Not a function call. *)
+           FieldAccess rdot
+       | _ ->
+           let args : arguments =
+             match v2 with
+             | Some x -> map_call_arguments_without_parentheses env x
+             | None -> ([], [])
+           in
+           Call (mk_call_no_parens (Right rdot) args blopt))
 
 and map_capture_expression (env : env) (x : CST.capture_expression) =
   match x with
@@ -668,7 +674,7 @@ and map_capture_expression (env : env) (x : CST.capture_expression) =
               | other -> other
             in
             (match actual_fun_name with
-            | I _ | Alias _ | DotAlias _ | DotRemote _ ->
+            | I _ | Alias _ | DotAlias _ | DotRemote _ | FieldAccess _ ->
                 (* Convert &fun/arity to &(fun(&1, &2, ...)) *)
                 let arity_int = Int64.to_int arity in
                 (* Create PlaceHolder arguments: &1, &2, ... *)
@@ -884,9 +890,7 @@ and map_expression (env : env) (x : CST.expression) : expr =
   | `Un_op x -> map_unary_operator env x
   | `Bin_op x -> map_binary_operator env x
   | `Dot x -> map_dot env x
-  | `Call x ->
-      let c = map_call env x in
-      Call c
+  | `Call x -> map_call env x
   (* semantic: transformed in Access.get/2 *)
   | `Access_call (v1, v2, v3, v4) ->
       let v1 = map_expression env v1 in

languages/lisp/tree-sitter/Parse_clojure_tree_sitter.ml

@@ -1111,17 +1111,24 @@ and map_binding_form_map_lit (env : env) ((_meta, (lb, srcs, rb)) : CST.map_lit)
     in
     with_or_as s (token env tk) pats rest
 
-  (* Standard map binding, eg, {x :a, [y z] :b}. *)
-  | _bind_form :: `Kwd_lit _  :: _ ->
-    let rec keyval_and_rest acc = function
-      | bind_form :: `Kwd_lit kwd_lit  :: rest_forms  ->
-        let key = map_binding_form env bind_form in
-        (* TODO: PatRecord of (dotted_ident * pattern) list bracket *)
+  (* Standard map binding, eg, {x :a, [y z] :b, z "str-key"}. *)
+  | _bind_form :: (`Kwd_lit _ | `Str_lit _) :: _ ->
+    let map_value_key_pattern = function
+      | `Kwd_lit kwd_lit ->
         let atom_kind, tok_colon, atom_name =
           map_kwd_expr_aux env kwd_lit
         in
-        let value = G.OtherPat ((atom_kind, tok_colon), [G.Name atom_name]) in
-        (* let value = G.PatLiteral (map_kwd_lit env kwd_lit) in *)
+        G.OtherPat ((atom_kind, tok_colon), [G.Name atom_name])
+      | `Str_lit str_tok ->
+        let s, t = H.str env str_tok in
+        let s_no_quotes = String.sub s 1 (String.length s - 2) in
+        G.PatLiteral (G.String (Tok.unsafe_fake_bracket (s_no_quotes, t)))
+    in
+    let rec keyval_and_rest acc = function
+      | bind_form :: (`Kwd_lit _ | `Str_lit _ as kv) :: rest_forms ->
+        let key = map_binding_form env bind_form in
+        (* TODO: PatRecord of (dotted_ident * pattern) list bracket *)
+        let value = map_value_key_pattern kv in
         keyval_and_rest
           (G.PatKeyVal (key, value) :: acc)
           rest_forms

libs/ast_generic/AST_generic_helpers.ml

@@ -199,9 +199,7 @@ let rec expr_to_pattern e =
   | Container (Tuple, (t1, xs, t2)) ->
       PatTuple (t1, xs |> List_.map expr_to_pattern, t2)
   | L l -> PatLiteral l
-  | Container (List, (t1, xs, t2)) ->
-      PatList (t1, xs |> List_.map expr_to_pattern, t2)
-  | Container (Dict, (t1, xs, t2)) ->
+  | Container ((List | Dict), (t1, xs, t2)) ->
       PatList (t1, xs |> List_.map expr_to_pattern, t2)
   | Constructor (n, (_, args, _)) ->
       PatConstructor (n, args |> List_.map expr_to_pattern)

scripts/build-nuitka.sh

@@ -60,6 +60,7 @@ pushd cli
   --include-data-dir="$SRC_SEMGREP_DIR/templates=semgrep/templates" \
   --include-data-file="$SRC_SEMGREP_DIR/semgrep_interfaces/lang.json=semgrep/semgrep_interfaces/lang.json" \
   --include-data-file="$SRC_SEMGREP_DIR/semgrep_interfaces/rule_schema_v1.yaml=semgrep/semgrep_interfaces/rule_schema_v1.yaml" \
+  --include-package=charset_normalizer \
   --no-deployment-flag=self-execution \
   --windows-icon-from-ico=spec/opengrep.ico \
   --linux-icon=spec/opengrep.ico \