Merge branch 'kn-51937-extra-object-key-pass' into 'master'

kn-51937-extra-object-key-pass

See merge request flussonic/openapi_handler!59

Author: Yuriy Zhloba

.gitlab-ci.yml

@@ -22,3 +22,19 @@ run-test:
     expire_in: 1 week
     paths:
       - test-logs
+
+run-test-legacy:
+  stage: test
+  needs:
+    - prepare-src
+  script:
+    - mkdir -p test-logs-legacy
+    - docker run --rm -v ${PWD}/test-logs-legacy:/openapi_handler/test-logs openapi_handler make ci-test-legacy
+  artifacts:
+    reports:
+      junit: test-logs-legacy/*/junit_report.xml
+    when: always
+    name: "${CI_JOB_STAGE}_${BRANCH_NAME}-test-legacy"
+    expire_in: 1 week
+    paths:
+      - test-logs-legacy

Makefile

@@ -6,5 +6,8 @@ all:
 ci-test:
 	REBAR_CONFIG=rebar.config_ ./rebar3 as dev ct --logdir test-logs --readable true
 
+ci-test-legacy:
+	REBAR_CONFIG=rebar.config_ ./rebar3 as dev_legacy ct --logdir test-logs --readable true
+
 clean:
 	rm -rf _build rebar.lock

README.md

@@ -87,6 +87,7 @@ Also see validation quirks below.
 You can customize the way objects are processed. 
 Available options:
  * `#{extra_obj_key => drop}` -- the original object may contain keys not described by schema, just ignore them instead of raising an error
+ * `#{extra_obj_key => pass}` -- the original object may contain keys not described by schema, just pass them as is instead of raising an error
  * `#{required_obj_keys => error}` -- raise an error when original object misses some required keys
  * `#{validators => #{Format :: atom() => Validator}}`  
    `Validator :: fun(Value) -> {ok, ConvertedValue} | {error, #{}}`  

rebar.config_

@@ -12,5 +12,16 @@
       redbug
     ]},
     {ct_opts, [{ct_hooks, [cth_surefire]}]}
+  ]},
+  {dev_legacy, [
+    {deps, [
+      jsx,
+      yamerl,
+      {cowboy, "1.0.1"},
+      {lhttpc, {git, "https://github.com/erlyvideo/lhttpc.git", {branch, "master"}}},
+      redbug
+    ]},
+    {erl_opts, [{d,legacy}]},
+    {ct_opts, [{ct_hooks, [cth_surefire]}]}
   ]}
 ]}.

src/openapi_handler.erl

@@ -3,7 +3,7 @@
 
 
 -export([init/2, handle/2, terminate/3]).
--export([routes/1, load_schema/2]).
+-export([routes/1, load_schema/2, choose_module/0]).
 -export([read_schema/1]).
 -export([routes_sort/1]). % for tests
 
@@ -14,6 +14,8 @@
 routes(#{schema := SchemaPath, module := Module, name := Name, prefix := Prefix} = Config) ->
   #{} = Schema = load_schema(SchemaPath, Name),
   SchemaOpts = get_schema_opts(Config),
+  HandlerModule = choose_module(),
+
   #{paths := Paths} = Schema,
   Routes = lists:map(fun({Path,PathSpec}) ->
     <<"/", _/binary>> = CowboyPath = re:replace(atom_to_list(Path), "{([^\\}]+)}", ":\\1",[global,{return,binary}]),
@@ -25,13 +27,23 @@ routes(#{schema := SchemaPath, module := Module, name := Name, prefix := Prefix}
     % It is too bad to pass all this stuff through cowboy options because it starts suffering
     % from GC on big state. Either ETS, either persistent_term, either compilation of custom code
     persistent_term:put({openapi_handler_route,Name,CowboyPath}, PathSpec1#{name => Name, module => Module, schema_opts => SchemaOpts}),
-    {<<Prefix/binary, CowboyPath/binary>>, ?MODULE, {Name,CowboyPath}}
+    {<<Prefix/binary, CowboyPath/binary>>, HandlerModule, {Name,CowboyPath}}
   end, maps:to_list(Paths)),
   % After sorting, bindings (starting with ":") must be after constant path segments, so generic routes only work after specific ones.
   % E.g. "/api/users/admin" must be before "/api/users/:id"
   % Thus special sorting function
   routes_sort(Routes).
 
+choose_module() ->
+  try
+    % Check if cowboy uses modern Req (map)
+    1234 = cowboy_req:port(#{port => 1234}),
+    ?MODULE
+  catch
+    error:{badrecord,_}:_ ->
+      % cowboy uses record for Req, use legacy wrapper
+      openapi_handler_legacy
+  end.
 
 prepare_operation_fm(_M, #{operationId := OperationId_} = Operation, Parameters) ->
   Op1 = maps:remove(description,Operation),
@@ -528,7 +540,7 @@ terminate(_,_,_) ->
 
 get_schema_opts(#{schema_opts := #{} = SchemaOpts}) ->
   maps:map(fun
-    (extra_obj_key,Flag) when Flag == drop; Flag == error -> ok;
+    (extra_obj_key,Flag) when Flag == drop; Flag == error; Flag == pass -> ok;
     (K,V) -> error({bad_schema_opt,K,V})
   end, SchemaOpts),
   SchemaOpts;

src/openapi_handler_legacy.erl

@@ -0,0 +1,92 @@
+-module(openapi_handler_legacy).
+
+-export([init/3, handle/2, terminate/3]).
+
+-export([method/1, peer/1, qs/1, parse_qs/1, bindings/1]).
+-export([read_body/1, reply/4, headers/1, header/2, header/3, parse_header/2]).
+-export([read_multipart_files/1]).
+
+init(_, Req, {Name, CowboyPath}) ->
+  openapi_handler:do_init(Req, Name, CowboyPath, ?MODULE, #{ok => shutdown, no_handle => true}).
+
+handle(Req, #{} = Request) ->
+  openapi_handler:do_handle(Req, Request, ?MODULE).
+
+terminate(_,_,_) ->
+  ok.
+
+
+%% Cowboy compat wrapper. Provides Cowboy 2.9 API for Cowboy 1.0
+method(Req) ->
+  {Method, _Req1} = cowboy_req:method(Req),
+  Method.
+
+peer(Req) ->
+  {Peer, _Req1} = cowboy_req:peer(Req),
+  Peer.
+
+qs(Req) ->
+  {QS, _Req1} = cowboy_req:qs(Req),
+  QS.
+
+parse_qs(Req) ->
+  try
+    cow_qs:parse_qs(qs(Req))
+  catch _:_:Stacktrace ->
+    erlang:raise(exit, {request_error, qs,
+      'Malformed query string; application/x-www-form-urlencoded expected.'
+      }, Stacktrace)
+  end.
+
+bindings(Req) ->
+  {Bindings, _Req1} = cowboy_req:bindings(Req),
+  maps:from_list(Bindings).
+
+read_body(Req) ->
+  cowboy_req:body(Req).
+
+reply(Code, Headers, Body, Req) ->
+  {ok, Req1} = cowboy_req:reply(Code, maps:to_list(Headers), Body, Req),
+  Req1.
+
+headers(Req) ->
+  {Headers, _Req1} = cowboy_req:headers(Req),
+  maps:from_list(Headers).
+
+header(Name, Req) ->
+  {Value, _Req1} = cowboy_req:header(Name, Req),
+  Value.
+
+header(Name, Req, Default) ->
+  {Value, _Req1} = cowboy_req:header(Name, Req, Default),
+  Value.
+
+parse_header(Name, Req) ->
+  {ok, Value, _Req1} = cowboy_req:parse_header(Name, Req),
+  Value.
+
+
+
+read_multipart_files(Req) ->
+  % This is a copy of openapi_handler:read_multipart_files/1 with
+  % cowboy_req:part/1, cowboy_req:part_body/1 and different number of cow_multipart:form_data/1
+  % results.
+  % Despite other methods are cowboy_req methods in this module, it is much simpler to keep such
+  % method instead of multiple call mocks to read_part/1 and read_part_body/1 in tests
+  do_read_multipart_files(Req, []).
+
+do_read_multipart_files(Req0, Files) ->
+  case cowboy_req:part(Req0) of
+    {ok, Headers, Req1} ->
+      {file, _FieldName, Filename, _CType, _TE} = cow_multipart:form_data(Headers),
+      {Bin, Req2} = read_multipart_file(Req1, <<>>),
+      do_read_multipart_files(Req2, [{Filename, Bin}| Files]);
+    {done, Req1} ->
+      {ok, Files, Req1}
+end.
+
+read_multipart_file(Req0, Bin) ->
+  case cowboy_req:part_body(Req0) of
+    {ok, LastBodyChunk, Req} -> {<<Bin/binary, LastBodyChunk/binary>>, Req};
+    {more, BodyChunk, Req} -> read_multipart_file(Req, <<Bin/binary, BodyChunk/binary>>)
+  end.

src/openapi_schema.erl

@@ -45,7 +45,7 @@ process(Input, #{} = Opts) ->
     (query,Flag) when Flag == true; Flag == false -> ok;
     (apply_defaults,Flag) when Flag == true; Flag == false -> ok;
     (patch,Flag) when Flag == true; Flag == false -> ok;
-    (extra_obj_key,Flag) when Flag == drop; Flag == error -> ok;
+    (extra_obj_key,Flag) when Flag == drop; Flag == error; Flag == pass -> ok;
     (required_obj_keys,Flag) when Flag == drop; Flag == error -> ok;
     (access_type,Flag) when Flag == read; Flag == write -> ok;
     (explain,FlagList) -> check_explain_keys(FlagList);
@@ -593,6 +593,11 @@ check_extra_keys(Input, Encoded, #{extra_obj_key := error} = Opts) when is_map(I
     _ -> Encoded
   end;
 
+check_extra_keys(Input, Encoded, #{extra_obj_key := pass}) when is_map(Input) andalso is_map(Encoded) andalso map_size(Encoded) > 0 ->
+  EncodedKeys = [K || K <- maps:keys(Encoded), is_atom(K)],
+  ExtraKeys = maps:keys(Input) -- [atom_to_binary(K) || K <- EncodedKeys],
+  maps:merge(maps:with(EncodedKeys, Encoded), maps:with(ExtraKeys, Input));
+
 check_extra_keys(_Input, Encoded, _Opts) ->
   Encoded.
 

test/openapi_handler_SUITE.erl

@@ -36,10 +36,15 @@ groups() ->
    ]}
   ].
 
+-ifdef(legacy).
+start_http(Routes, ApiName) ->
+  cowboy:start_http(ApiName, 1, [{port, 0}],
+    [{env, [{dispatch, cowboy_router:compile([{'_', Routes}])}]}]).
+-else.
 start_http(Routes, ApiName) ->
   cowboy:start_clear(ApiName, [{port, 0}],
     #{env => #{dispatch => cowboy_router:compile([{'_', Routes}])}}).
-
+-endif.
 
 init_per_suite(Config) ->
   {ok, _} = application:ensure_all_started(cowboy),
@@ -280,7 +285,7 @@ json_array_ok(_) ->
   ok.
 
 putFile(#{req := Req, '$cowboy_req' := CowboyReq}) ->
-  <<"PUT">> = cowboy_req:method(CowboyReq),
+  <<"PUT">> = ?MODULE:method(CowboyReq),
   Body = <<"{\"size\":100}">>,
   Req1 = reply(200, #{<<"content-length">> => byte_size(Body), <<"content-type">> => <<"application/json">>}, Body, Req),
   {done, Req1}.

test/openapi_schema_SUITE.erl

@@ -13,6 +13,7 @@ groups() ->
       read_default,
       extra_keys_error,
       extra_keys_drop,
+      extra_keys_pass,
       null_in_array,
       nullable_by_oneof,
       discriminator,
@@ -84,6 +85,14 @@ extra_keys_drop(_) ->
   #{inputs := [],name := <<"read_default">>,static := true} = openapi_schema:process(Json, #{type => stream_config, whole_schema => Schema, apply_defaults => true}),
   ok.
 
+extra_keys_pass(_) ->
+  Json = #{<<"name">> => <<"read_default">>, extra_key1 => <<"abc">>, <<"extrakey2">> => def},
+  Schema = persistent_term:get({openapi_handler_schema,test_openapi}),
+  #{inputs := [],name := <<"read_default">>,static := true,
+      extra_key1 := <<"abc">>,<<"extrakey2">> := def
+   } = openapi_schema:process(Json, #{type => stream_config, whole_schema => Schema, apply_defaults => true, extra_obj_key => pass}),
+  ok.
+
 
 null_in_array(_) ->
   % When items are not nullable, passing null|undefined as a list element should return an error