From 0bd2c08bce02c57bc5907a0e2d1b486fb88a356e Mon Sep 17 00:00:00 2001 From: Oleg Date: Fri, 6 Dec 2024 16:21:33 +0100 Subject: [PATCH] chore: update editorconfig to match the current project style --- .editorconfig | 6 +- src/ar_bundles.erl | 114 +-- src/ar_deep_hash.erl | 18 +- src/ar_tx.erl | 210 +++--- src/ar_wallet.erl | 240 +++---- src/dev_cu.erl | 18 +- src/dev_message.erl | 204 +++--- src/dev_meta.erl | 56 +- src/dev_mu.erl | 118 ++-- src/dev_multipass.erl | 2 +- src/dev_p4.erl | 2 +- src/dev_poda.erl | 34 +- src/dev_scheduler.erl | 20 +- src/dev_scheduler_interface.erl | 8 +- src/dev_scheduler_registry.erl | 6 +- src/dev_scheduler_server.erl | 12 +- src/dev_stack.erl | 270 ++++---- src/dev_vfs.erl | 2 +- src/dev_wasm.erl | 2 +- src/hb.app.src | 40 +- src/hb.erl | 34 +- src/hb_app.erl | 2 +- src/hb_beamr.erl | 24 +- src/hb_cache.erl | 30 +- src/hb_client.erl | 50 +- src/hb_crypto.erl | 40 +- src/hb_http.erl | 18 +- src/hb_http_router.erl | 144 ++-- src/hb_http_signature.erl | 548 +++++++-------- src/hb_http_structured_fields.erl | 618 ++++++++--------- src/hb_message.erl | 1068 ++++++++++++++--------------- src/hb_metrics_collector.erl | 80 +-- src/hb_opts.erl | 172 ++--- src/hb_pam.erl | 942 ++++++++++++------------- src/hb_path.erl | 246 +++---- src/hb_private.erl | 54 +- src/hb_process.erl | 192 +++--- src/hb_process_monitor.erl | 6 +- src/hb_router.erl | 2 +- src/hb_store.erl | 56 +- src/hb_store_fs.erl | 4 +- src/hb_store_remote_node.erl | 2 +- src/hb_sup.erl | 6 +- src/hb_test.erl | 2 +- src/hb_util.erl | 354 +++++----- src/include/ar.hrl | 106 +-- src/include/hb.hrl | 10 +- src/include/hb_http.hrl | 74 +- src/rsa_pss.erl | 302 ++++---- src/sec.erl | 112 +-- src/sec_helpers.erl | 34 +- src/sec_tee.erl | 4 +- src/sec_tpm.erl | 222 +++--- 53 files changed, 3470 insertions(+), 3470 deletions(-) diff --git a/.editorconfig b/.editorconfig index a1ececdb..f26a02c9 100644 --- a/.editorconfig +++ b/.editorconfig @@ -7,11 +7,11 @@ charset = utf-8 end_of_line = lf -trim_trailing_whitespace = true +trim_trailing_whitespace = false -insert_final_newline = true +insert_final_newline = false -indent_style = tab +indent_style = space indent_size = 4 [*.{js,json,yaml,yml}] diff --git a/src/ar_bundles.erl b/src/ar_bundles.erl index ca4abd1b..72aa7a36 100644 --- a/src/ar_bundles.erl +++ b/src/ar_bundles.erl @@ -65,11 +65,11 @@ format(Item, Indent) when is_record(Item, tx) -> format_line("!!! CAUTION: ITEM IS SIGNED BUT INVALID !!!", Indent + 1); false -> [] end ++ - case is_signed(Item) of - true -> - format_line("Signer: ~s", [hb_util:encode(signer(Item))], Indent + 1); - false -> [] - end ++ + case is_signed(Item) of + true -> + format_line("Signer: ~s", [hb_util:encode(signer(Item))], Indent + 1); + false -> [] + end ++ format_line("Target: ~s", [ case Item#tx.target of <<>> -> "[NONE]"; @@ -276,7 +276,7 @@ data_item_signature_data(RawItem) -> data_item_signature_data(RawItem, unsigned) -> data_item_signature_data(RawItem#tx { owner = ?DEFAULT_OWNER }, signed); data_item_signature_data(RawItem, signed) -> - true = enforce_valid_tx(RawItem), + true = enforce_valid_tx(RawItem), NormItem = normalize_data(RawItem), ar_deep_hash:hash([ utf8_encoded("dataitem"), @@ -371,7 +371,7 @@ serialize(not_found) -> throw(not_found); serialize(TX) -> serialize(TX, binary). serialize(TX, binary) when is_binary(TX) -> TX; serialize(RawTX, binary) -> - true = enforce_valid_tx(RawTX), + true = enforce_valid_tx(RawTX), TX = normalize(RawTX), EncodedTags = encode_tags(TX#tx.tags), << @@ -385,7 +385,7 @@ serialize(RawTX, binary) -> (TX#tx.data)/binary >>; serialize(TX, json) -> - true = enforce_valid_tx(TX), + true = enforce_valid_tx(TX), jiffy:encode(item_to_json_struct(TX)). %% @doc Take an item and ensure that it is of valid form. Useful for ensuring @@ -393,11 +393,11 @@ serialize(TX, json) -> %% This function should throw simple, easy to follow errors to aid devs in %% debugging issues. enforce_valid_tx(List) when is_list(List) -> - lists:all(fun enforce_valid_tx/1, List); + lists:all(fun enforce_valid_tx/1, List); enforce_valid_tx(Map) when is_map(Map) -> - lists:all(fun(Item) -> enforce_valid_tx(Item) end, maps:values(Map)); + lists:all(fun(Item) -> enforce_valid_tx(Item) end, maps:values(Map)); enforce_valid_tx(TX) -> - ok_or_throw(TX, + ok_or_throw(TX, check_type(TX, message), {invalid_tx, TX} ), @@ -413,60 +413,60 @@ enforce_valid_tx(TX) -> check_size(TX#tx.last_tx, [0, 32]), {invalid_field, last_tx, TX#tx.last_tx} ), - ok_or_throw(TX, - check_size(TX#tx.owner, [0, byte_size(?DEFAULT_OWNER)]), - {invalid_field, owner, TX#tx.owner} - ), + ok_or_throw(TX, + check_size(TX#tx.owner, [0, byte_size(?DEFAULT_OWNER)]), + {invalid_field, owner, TX#tx.owner} + ), ok_or_throw(TX, check_size(TX#tx.target, [0, 32]), {invalid_field, target, TX#tx.target} ), - ok_or_throw(TX, - check_size(TX#tx.signature, [0, byte_size(?DEFAULT_SIG)]), - {invalid_field, signature, TX#tx.signature} - ), - lists:foreach( - fun({Name, Value}) -> - ok_or_throw(TX, - check_type(Name, binary), - {invalid_field, tag_name, Name} - ), - ok_or_throw(TX, - check_size(Name, {range, 0, ?MAX_TAG_NAME_SIZE}), - {invalid_field, tag_name, Name} - ), - ok_or_throw(TX, - check_type(Value, binary), - {invalid_field, tag_value, Value} - ), - ok_or_throw(TX, - check_size(Value, {range, 0, ?MAX_TAG_VALUE_SIZE}), - {invalid_field, tag_value, Value} - ); - (InvalidTagForm) -> - throw({invalid_field, tag, InvalidTagForm}) - end, - TX#tx.tags - ), - ok_or_throw( - TX, - check_type(TX#tx.data, binary) - orelse check_type(TX#tx.data, map) - orelse check_type(TX#tx.data, list), - {invalid_field, data, TX#tx.data} - ), - true. + ok_or_throw(TX, + check_size(TX#tx.signature, [0, byte_size(?DEFAULT_SIG)]), + {invalid_field, signature, TX#tx.signature} + ), + lists:foreach( + fun({Name, Value}) -> + ok_or_throw(TX, + check_type(Name, binary), + {invalid_field, tag_name, Name} + ), + ok_or_throw(TX, + check_size(Name, {range, 0, ?MAX_TAG_NAME_SIZE}), + {invalid_field, tag_name, Name} + ), + ok_or_throw(TX, + check_type(Value, binary), + {invalid_field, tag_value, Value} + ), + ok_or_throw(TX, + check_size(Value, {range, 0, ?MAX_TAG_VALUE_SIZE}), + {invalid_field, tag_value, Value} + ); + (InvalidTagForm) -> + throw({invalid_field, tag, InvalidTagForm}) + end, + TX#tx.tags + ), + ok_or_throw( + TX, + check_type(TX#tx.data, binary) + orelse check_type(TX#tx.data, map) + orelse check_type(TX#tx.data, list), + {invalid_field, data, TX#tx.data} + ), + true. %% @doc Force that a binary is either empty or the given number of bytes. check_size(Bin, {range, Start, End}) -> - check_type(Bin, binary) - andalso byte_size(Bin) >= Start - andalso byte_size(Bin) =< End; + check_type(Bin, binary) + andalso byte_size(Bin) >= Start + andalso byte_size(Bin) =< End; check_size(Bin, X) when not is_list(X) -> check_size(Bin, [X]); check_size(Bin, Sizes) -> - check_type(Bin, binary) - andalso lists:member(byte_size(Bin), Sizes). + check_type(Bin, binary) + andalso lists:member(byte_size(Bin), Sizes). %% @doc Ensure that a value is of the given type. check_type(Value, binary) when is_binary(Value) -> true; @@ -476,13 +476,13 @@ check_type(Value, _) when is_list(Value) -> false; check_type(Value, map) when is_map(Value) -> true; check_type(Value, _) when is_map(Value) -> false; check_type(Value, message) -> - is_record(Value, tx) or is_map(Value) or is_list(Value); + is_record(Value, tx) or is_map(Value) or is_list(Value); check_type(_Value, _) -> false. %% @doc Throw an error if the given value is not ok. ok_or_throw(_, true, _) -> true; ok_or_throw(_TX, false, Error) -> - throw(Error). + throw(Error). %% @doc Take an item and ensure that both the unsigned and signed IDs are %% appropriately set. This function is structured to fall through all cases diff --git a/src/ar_deep_hash.erl b/src/ar_deep_hash.erl index 3fdfd7b3..70e80e3e 100644 --- a/src/ar_deep_hash.erl +++ b/src/ar_deep_hash.erl @@ -6,18 +6,18 @@ hash(List) when is_list(List) -> hash_bin_or_list(List). %%% INTERNAL hash_bin_or_list(Bin) when is_binary(Bin) -> - Tag = <<"blob", (integer_to_binary(byte_size(Bin)))/binary>>, - hash_bin(<<(hash_bin(Tag))/binary, (hash_bin(Bin))/binary>>); + Tag = <<"blob", (integer_to_binary(byte_size(Bin)))/binary>>, + hash_bin(<<(hash_bin(Tag))/binary, (hash_bin(Bin))/binary>>); hash_bin_or_list(List) when is_list(List) -> - Tag = <<"list", (integer_to_binary(length(List)))/binary>>, - hash_list(List, hash_bin(Tag)). + Tag = <<"list", (integer_to_binary(length(List)))/binary>>, + hash_list(List, hash_bin(Tag)). hash_list([], Acc) -> - Acc; + Acc; hash_list([Head | List], Acc) -> - HashPair = <>, - NewAcc = hash_bin(HashPair), - hash_list(List, NewAcc). + HashPair = <>, + NewAcc = hash_bin(HashPair), + hash_list(List, NewAcc). hash_bin(Bin) when is_binary(Bin) -> - crypto:hash(sha384, Bin). + crypto:hash(sha384, Bin). \ No newline at end of file diff --git a/src/ar_tx.erl b/src/ar_tx.erl index 45efd42b..8b77d241 100644 --- a/src/ar_tx.erl +++ b/src/ar_tx.erl @@ -102,110 +102,110 @@ collect_validation_results(_TXID, Checks) -> end. json_struct_to_tx(TXStruct) -> - Tags = - case hb_util:find_value(<<"tags">>, TXStruct) of - undefined -> - []; - Xs -> - Xs - end, - Data = hb_util:decode(hb_util:find_value(<<"data">>, TXStruct)), - Format = - case hb_util:find_value(<<"format">>, TXStruct) of - undefined -> - 1; - N when is_integer(N) -> - N; - N when is_binary(N) -> - binary_to_integer(N) - end, - Denomination = - case hb_util:find_value(<<"denomination">>, TXStruct) of - undefined -> - 0; - EncodedDenomination -> - MaybeDenomination = binary_to_integer(EncodedDenomination), - true = MaybeDenomination > 0, - MaybeDenomination - end, - TXID = hb_util:decode(hb_util:find_value(<<"id">>, TXStruct)), - 32 = byte_size(TXID), - #tx{ - format = Format, - id = TXID, - last_tx = hb_util:decode(hb_util:find_value(<<"last_tx">>, TXStruct)), - owner = hb_util:decode(hb_util:find_value(<<"owner">>, TXStruct)), - tags = [{hb_util:decode(Name), hb_util:decode(Value)} - %% Only the elements matching this pattern are included in the list. - || {[{<<"name">>, Name}, {<<"value">>, Value}]} <- Tags], - target = hb_util:find_value(<<"target">>, TXStruct), - quantity = binary_to_integer(hb_util:find_value(<<"quantity">>, TXStruct)), - data = Data, - reward = binary_to_integer(hb_util:find_value(<<"reward">>, TXStruct)), - signature = hb_util:decode(hb_util:find_value(<<"signature">>, TXStruct)), - data_size = binary_to_integer(hb_util:find_value(<<"data_size">>, TXStruct)), - data_root = - case hb_util:find_value(<<"data_root">>, TXStruct) of - undefined -> <<>>; - DR -> hb_util:decode(DR) - end, - denomination = Denomination - }. + Tags = + case hb_util:find_value(<<"tags">>, TXStruct) of + undefined -> + []; + Xs -> + Xs + end, + Data = hb_util:decode(hb_util:find_value(<<"data">>, TXStruct)), + Format = + case hb_util:find_value(<<"format">>, TXStruct) of + undefined -> + 1; + N when is_integer(N) -> + N; + N when is_binary(N) -> + binary_to_integer(N) + end, + Denomination = + case hb_util:find_value(<<"denomination">>, TXStruct) of + undefined -> + 0; + EncodedDenomination -> + MaybeDenomination = binary_to_integer(EncodedDenomination), + true = MaybeDenomination > 0, + MaybeDenomination + end, + TXID = hb_util:decode(hb_util:find_value(<<"id">>, TXStruct)), + 32 = byte_size(TXID), + #tx{ + format = Format, + id = TXID, + last_tx = hb_util:decode(hb_util:find_value(<<"last_tx">>, TXStruct)), + owner = hb_util:decode(hb_util:find_value(<<"owner">>, TXStruct)), + tags = [{hb_util:decode(Name), hb_util:decode(Value)} + %% Only the elements matching this pattern are included in the list. + || {[{<<"name">>, Name}, {<<"value">>, Value}]} <- Tags], + target = hb_util:find_value(<<"target">>, TXStruct), + quantity = binary_to_integer(hb_util:find_value(<<"quantity">>, TXStruct)), + data = Data, + reward = binary_to_integer(hb_util:find_value(<<"reward">>, TXStruct)), + signature = hb_util:decode(hb_util:find_value(<<"signature">>, TXStruct)), + data_size = binary_to_integer(hb_util:find_value(<<"data_size">>, TXStruct)), + data_root = + case hb_util:find_value(<<"data_root">>, TXStruct) of + undefined -> <<>>; + DR -> hb_util:decode(DR) + end, + denomination = Denomination + }. tx_to_json_struct( - #tx{ - id = ID, - format = Format, - last_tx = Last, - owner = Owner, - tags = Tags, - target = Target, - quantity = Quantity, - data = Data, - reward = Reward, - signature = Sig, - data_size = DataSize, - data_root = DataRoot, - denomination = Denomination - }) -> - Fields = [ - {format, - case Format of - undefined -> - 1; - _ -> - Format - end}, - {id, hb_util:encode(ID)}, - {last_tx, hb_util:encode(Last)}, - {owner, hb_util:encode(Owner)}, - {tags, - lists:map( - fun({Name, Value}) -> - { - [ - {name, hb_util:encode(Name)}, - {value, hb_util:encode(Value)} - ] - } - end, - Tags - ) - }, - {target, hb_util:encode(Target)}, - {quantity, integer_to_binary(Quantity)}, - {data, hb_util:encode(Data)}, - {data_size, integer_to_binary(DataSize)}, - {data_tree, []}, - {data_root, hb_util:encode(DataRoot)}, - {reward, integer_to_binary(Reward)}, - {signature, hb_util:encode(Sig)} - ], - Fields2 = - case Denomination > 0 of - true -> - Fields ++ [{denomination, integer_to_binary(Denomination)}]; - false -> - Fields - end, - maps:from_list(Fields2). \ No newline at end of file + #tx{ + id = ID, + format = Format, + last_tx = Last, + owner = Owner, + tags = Tags, + target = Target, + quantity = Quantity, + data = Data, + reward = Reward, + signature = Sig, + data_size = DataSize, + data_root = DataRoot, + denomination = Denomination + }) -> + Fields = [ + {format, + case Format of + undefined -> + 1; + _ -> + Format + end}, + {id, hb_util:encode(ID)}, + {last_tx, hb_util:encode(Last)}, + {owner, hb_util:encode(Owner)}, + {tags, + lists:map( + fun({Name, Value}) -> + { + [ + {name, hb_util:encode(Name)}, + {value, hb_util:encode(Value)} + ] + } + end, + Tags + ) + }, + {target, hb_util:encode(Target)}, + {quantity, integer_to_binary(Quantity)}, + {data, hb_util:encode(Data)}, + {data_size, integer_to_binary(DataSize)}, + {data_tree, []}, + {data_root, hb_util:encode(DataRoot)}, + {reward, integer_to_binary(Reward)}, + {signature, hb_util:encode(Sig)} + ], + Fields2 = + case Denomination > 0 of + true -> + Fields ++ [{denomination, integer_to_binary(Denomination)}]; + false -> + Fields + end, + maps:from_list(Fields2). \ No newline at end of file diff --git a/src/ar_wallet.erl b/src/ar_wallet.erl index b350fd38..ef9ee47a 100644 --- a/src/ar_wallet.erl +++ b/src/ar_wallet.erl @@ -13,10 +13,10 @@ %%%=================================================================== new() -> - new({rsa, 65537}). + new({rsa, 65537}). new(KeyType = {KeyAlg, PublicExpnt}) when KeyType =:= {rsa, 65537} -> {[_, Pub], [_, Pub, Priv|_]} = {[_, Pub], [_, Pub, Priv|_]} - = crypto:generate_key(KeyAlg, {4096, PublicExpnt}), + = crypto:generate_key(KeyAlg, {4096, PublicExpnt}), {{KeyType, Priv, Pub}, {KeyType, Pub}}. %% @doc Sign some data with a private key. @@ -47,7 +47,7 @@ verify({{rsa, PublicExpnt}, Pub}, Data, Sig) %% @doc Generate an address from a public key. to_address(Pubkey) -> - to_address(Pubkey, ?DEFAULT_KEY_TYPE). + to_address(Pubkey, ?DEFAULT_KEY_TYPE). to_address(PubKey, {rsa, 65537}) when bit_size(PubKey) == 256 -> %% Small keys are not secure, nobody is using them, the clause %% is for backwards-compatibility. @@ -62,124 +62,124 @@ to_address(PubKey, {rsa, 65537}) -> new_keyfile(KeyType, WalletName) when is_list(WalletName) -> new_keyfile(KeyType, list_to_binary(WalletName)); new_keyfile(KeyType, WalletName) -> - {Pub, Priv, Key} = - case KeyType of - {?RSA_SIGN_ALG, PublicExpnt} -> - {[Expnt, Pb], [Expnt, Pb, Prv, P1, P2, E1, E2, C]} = - crypto:generate_key(rsa, {?RSA_PRIV_KEY_SZ, PublicExpnt}), - Ky = - jiffy:encode( - { - [ - {kty, <<"RSA">>}, - {ext, true}, - {e, hb_util:encode(Expnt)}, - {n, hb_util:encode(Pb)}, - {d, hb_util:encode(Prv)}, - {p, hb_util:encode(P1)}, - {q, hb_util:encode(P2)}, - {dp, hb_util:encode(E1)}, - {dq, hb_util:encode(E2)}, - {qi, hb_util:encode(C)} - ] - } - ), - {Pb, Prv, Ky}; - {?ECDSA_SIGN_ALG, secp256k1} -> - {OrigPub, Prv} = crypto:generate_key(ecdh, secp256k1), - <<4:8, PubPoint/binary>> = OrigPub, - PubPointMid = byte_size(PubPoint) div 2, - <> = PubPoint, - Ky = - jiffy:encode( - { - [ - {kty, <<"EC">>}, - {crv, <<"secp256k1">>}, - {x, hb_util:encode(X)}, - {y, hb_util:encode(Y)}, - {d, hb_util:encode(Prv)} - ] - } - ), - {compress_ecdsa_pubkey(OrigPub), Prv, Ky}; - {?EDDSA_SIGN_ALG, ed25519} -> - {{_, Prv, Pb}, _} = new(KeyType), - Ky = - jiffy:encode( - { - [ - {kty, <<"OKP">>}, - {alg, <<"EdDSA">>}, - {crv, <<"Ed25519">>}, - {x, hb_util:encode(Pb)}, - {d, hb_util:encode(Prv)} - ] - } - ), - {Pb, Prv, Ky} - end, - Filename = wallet_filepath(WalletName, Pub, KeyType), - filelib:ensure_dir(Filename), - file:write_file(Filename, Key), - {{KeyType, Priv, Pub}, {KeyType, Pub}}. + {Pub, Priv, Key} = + case KeyType of + {?RSA_SIGN_ALG, PublicExpnt} -> + {[Expnt, Pb], [Expnt, Pb, Prv, P1, P2, E1, E2, C]} = + crypto:generate_key(rsa, {?RSA_PRIV_KEY_SZ, PublicExpnt}), + Ky = + jiffy:encode( + { + [ + {kty, <<"RSA">>}, + {ext, true}, + {e, hb_util:encode(Expnt)}, + {n, hb_util:encode(Pb)}, + {d, hb_util:encode(Prv)}, + {p, hb_util:encode(P1)}, + {q, hb_util:encode(P2)}, + {dp, hb_util:encode(E1)}, + {dq, hb_util:encode(E2)}, + {qi, hb_util:encode(C)} + ] + } + ), + {Pb, Prv, Ky}; + {?ECDSA_SIGN_ALG, secp256k1} -> + {OrigPub, Prv} = crypto:generate_key(ecdh, secp256k1), + <<4:8, PubPoint/binary>> = OrigPub, + PubPointMid = byte_size(PubPoint) div 2, + <> = PubPoint, + Ky = + jiffy:encode( + { + [ + {kty, <<"EC">>}, + {crv, <<"secp256k1">>}, + {x, hb_util:encode(X)}, + {y, hb_util:encode(Y)}, + {d, hb_util:encode(Prv)} + ] + } + ), + {compress_ecdsa_pubkey(OrigPub), Prv, Ky}; + {?EDDSA_SIGN_ALG, ed25519} -> + {{_, Prv, Pb}, _} = new(KeyType), + Ky = + jiffy:encode( + { + [ + {kty, <<"OKP">>}, + {alg, <<"EdDSA">>}, + {crv, <<"Ed25519">>}, + {x, hb_util:encode(Pb)}, + {d, hb_util:encode(Prv)} + ] + } + ), + {Pb, Prv, Ky} + end, + Filename = wallet_filepath(WalletName, Pub, KeyType), + filelib:ensure_dir(Filename), + file:write_file(Filename, Key), + {{KeyType, Priv, Pub}, {KeyType, Pub}}. wallet_filepath(Wallet) -> - filename:join([?WALLET_DIR, binary_to_list(Wallet)]). + filename:join([?WALLET_DIR, binary_to_list(Wallet)]). wallet_filepath2(Wallet) -> - filename:join([?WALLET_DIR, binary_to_list(Wallet)]). + filename:join([?WALLET_DIR, binary_to_list(Wallet)]). %% @doc Read the keyfile for the key with the given address from disk. %% Return not_found if arweave_keyfile_[addr].json or [addr].json is not found %% in [data_dir]/?WALLET_DIR. load_key(Addr) -> - Path = hb_util:encode(Addr), - case filelib:is_file(Path) of - false -> - Path2 = wallet_filepath2(hb_util:encode(Addr)), - case filelib:is_file(Path2) of - false -> - not_found; - true -> - load_keyfile(Path2) - end; - true -> - load_keyfile(Path) - end. + Path = hb_util:encode(Addr), + case filelib:is_file(Path) of + false -> + Path2 = wallet_filepath2(hb_util:encode(Addr)), + case filelib:is_file(Path2) of + false -> + not_found; + true -> + load_keyfile(Path2) + end; + true -> + load_keyfile(Path) + end. %% @doc Extract the public and private key from a keyfile. load_keyfile(File) -> - {ok, Body} = file:read_file(File), - {Key} = jiffy:decode(Body), - {Pub, Priv, KeyType} = - case lists:keyfind(<<"kty">>, 1, Key) of - {<<"kty">>, <<"EC">>} -> - {<<"x">>, XEncoded} = lists:keyfind(<<"x">>, 1, Key), - {<<"y">>, YEncoded} = lists:keyfind(<<"y">>, 1, Key), - {<<"d">>, PrivEncoded} = lists:keyfind(<<"d">>, 1, Key), - OrigPub = iolist_to_binary([<<4:8>>, hb_util:decode(XEncoded), - hb_util:decode(YEncoded)]), - Pb = compress_ecdsa_pubkey(OrigPub), - Prv = hb_util:decode(PrivEncoded), - KyType = {?ECDSA_SIGN_ALG, secp256k1}, - {Pb, Prv, KyType}; - {<<"kty">>, <<"OKP">>} -> - {<<"x">>, PubEncoded} = lists:keyfind(<<"x">>, 1, Key), - {<<"d">>, PrivEncoded} = lists:keyfind(<<"d">>, 1, Key), - Pb = hb_util:decode(PubEncoded), - Prv = hb_util:decode(PrivEncoded), - KyType = {?EDDSA_SIGN_ALG, ed25519}, - {Pb, Prv, KyType}; - _ -> - {<<"n">>, PubEncoded} = lists:keyfind(<<"n">>, 1, Key), - {<<"d">>, PrivEncoded} = lists:keyfind(<<"d">>, 1, Key), - Pb = hb_util:decode(PubEncoded), - Prv = hb_util:decode(PrivEncoded), - KyType = {?RSA_SIGN_ALG, 65537}, - {Pb, Prv, KyType} - end, - {{KeyType, Priv, Pub}, {KeyType, Pub}}. + {ok, Body} = file:read_file(File), + {Key} = jiffy:decode(Body), + {Pub, Priv, KeyType} = + case lists:keyfind(<<"kty">>, 1, Key) of + {<<"kty">>, <<"EC">>} -> + {<<"x">>, XEncoded} = lists:keyfind(<<"x">>, 1, Key), + {<<"y">>, YEncoded} = lists:keyfind(<<"y">>, 1, Key), + {<<"d">>, PrivEncoded} = lists:keyfind(<<"d">>, 1, Key), + OrigPub = iolist_to_binary([<<4:8>>, hb_util:decode(XEncoded), + hb_util:decode(YEncoded)]), + Pb = compress_ecdsa_pubkey(OrigPub), + Prv = hb_util:decode(PrivEncoded), + KyType = {?ECDSA_SIGN_ALG, secp256k1}, + {Pb, Prv, KyType}; + {<<"kty">>, <<"OKP">>} -> + {<<"x">>, PubEncoded} = lists:keyfind(<<"x">>, 1, Key), + {<<"d">>, PrivEncoded} = lists:keyfind(<<"d">>, 1, Key), + Pb = hb_util:decode(PubEncoded), + Prv = hb_util:decode(PrivEncoded), + KyType = {?EDDSA_SIGN_ALG, ed25519}, + {Pb, Prv, KyType}; + _ -> + {<<"n">>, PubEncoded} = lists:keyfind(<<"n">>, 1, Key), + {<<"d">>, PrivEncoded} = lists:keyfind(<<"d">>, 1, Key), + Pb = hb_util:decode(PubEncoded), + Prv = hb_util:decode(PrivEncoded), + KyType = {?RSA_SIGN_ALG, 65537}, + {Pb, Prv, KyType} + end, + {{KeyType, Priv, Pub}, {KeyType, Pub}}. %%%=================================================================== %%% Private functions. @@ -196,19 +196,19 @@ hash_address(PubKey) -> %%%=================================================================== wallet_filepath(WalletName, PubKey, KeyType) -> - wallet_filepath(wallet_name(WalletName, PubKey, KeyType)). + wallet_filepath(wallet_name(WalletName, PubKey, KeyType)). wallet_name(wallet_address, PubKey, KeyType) -> - hb_util:encode(to_address(PubKey, KeyType)); + hb_util:encode(to_address(PubKey, KeyType)); wallet_name(WalletName, _, _) -> - WalletName. + WalletName. compress_ecdsa_pubkey(<<4:8, PubPoint/binary>>) -> - PubPointMid = byte_size(PubPoint) div 2, - <> = PubPoint, - PubKeyHeader = - case Y rem 2 of - 0 -> <<2:8>>; - 1 -> <<3:8>> - end, - iolist_to_binary([PubKeyHeader, X]). \ No newline at end of file + PubPointMid = byte_size(PubPoint) div 2, + <> = PubPoint, + PubKeyHeader = + case Y rem 2 of + 0 -> <<2:8>>; + 1 -> <<3:8>> + end, + iolist_to_binary([PubKeyHeader, X]). \ No newline at end of file diff --git a/src/dev_cu.erl b/src/dev_cu.erl index c30506ee..2ca70404 100644 --- a/src/dev_cu.erl +++ b/src/dev_cu.erl @@ -5,12 +5,12 @@ -hb_debug(print). push(Msg, S = #{ assignment := Assignment, logger := _Logger }) -> - ?event( - {pushing_message, - {assignment, hb_util:id(Assignment, unsigned)}, - {message, hb_util:id(Msg, unsigned)} - } - ), + ?event( + {pushing_message, + {assignment, hb_util:id(Assignment, unsigned)}, + {message, hb_util:id(Msg, unsigned)} + } + ), case hb_client:compute(Assignment, Msg) of {ok, Results} -> ?event(computed_results), @@ -28,13 +28,13 @@ execute(CarrierMsg, S) -> #tx{data = #{ <<"Message">> := _Msg, <<"Assignment">> := Assignment }} -> % TODO: Execute without needing to call the SU unnecessarily. {_, ProcID} = lists:keyfind(<<"Process">>, 1, Assignment#tx.tags), - ?event({dev_cu_computing_from_full_assignment, {process, ProcID}, {slot, hb_util:id(Assignment, signed)}}), + ?event({dev_cu_computing_from_full_assignment, {process, ProcID}, {slot, hb_util:id(Assignment, signed)}}), hb_process:result(ProcID, hb_util:id(Assignment, signed), Store, Wallet); _ -> case lists:keyfind(<<"Process">>, 1, CarrierMsg#tx.tags) of {_, Process} -> {_, Slot} = lists:keyfind(<<"Slot">>, 1, CarrierMsg#tx.tags), - ?event({dev_cu_computing_from_slot_ref, {process, Process}, {slot, Slot}}), + ?event({dev_cu_computing_from_slot_ref, {process, Process}, {slot, Slot}}), hb_process:result(Process, Slot, Store, Wallet); false -> {error, no_viable_computation} @@ -76,5 +76,5 @@ execute(CarrierMsg, S) -> {ok, S#{ results => Results }} end, ?event(returning_computed_results), - %ar_bundles:print(ModResults), + %ar_bundles:print(ModResults), {ResType, ModState}. \ No newline at end of file diff --git a/src/dev_message.erl b/src/dev_message.erl index 2623d351..79faca08 100644 --- a/src/dev_message.erl +++ b/src/dev_message.erl @@ -12,171 +12,171 @@ %% @doc Return the info for the identity device. info() -> - #{ - default => fun get/3, - exports => ?DEVICE_KEYS - }. + #{ + default => fun get/3, + exports => ?DEVICE_KEYS + }. %% @doc Return the ID of a message. If the message already has an ID, return %% that. Otherwise, return the signed ID. id(M) -> - {ok, raw_id(M, signed)}. + {ok, raw_id(M, signed)}. %% @doc Wrap a call to the `hb_util:id/2` function, which returns the %% unsigned ID of a message. unsigned_id(M) -> - {ok, raw_id(M, unsigned)}. + {ok, raw_id(M, unsigned)}. %% @doc Encode an ID in any format to a normalized, b64u 43 character binary. raw_id(Item) -> raw_id(Item, unsigned). raw_id(TX, Type) when is_record(TX, tx) -> - hb_util:encode(ar_bundles:id(TX, Type)); + hb_util:encode(ar_bundles:id(TX, Type)); raw_id(Map, Type) when is_map(Map) -> - Msg = hb_message:message_to_tx(Map), - hb_util:encode(ar_bundles:id(Msg, Type)); + Msg = hb_message:message_to_tx(Map), + hb_util:encode(ar_bundles:id(Msg, Type)); raw_id(Bin, _) when is_binary(Bin) andalso byte_size(Bin) == 43 -> - Bin; + Bin; raw_id(Bin, _) when is_binary(Bin) andalso byte_size(Bin) == 32 -> - hb_util:encode(Bin); + hb_util:encode(Bin); raw_id(Data, _) when is_list(Data) -> - raw_id(list_to_binary(Data)). + raw_id(list_to_binary(Data)). %% @doc Return the signers of a message. signers(M) -> - {ok, hb_util:list_to_numbered_map(hb_message:signers(M))}. + {ok, hb_util:list_to_numbered_map(hb_message:signers(M))}. %% @doc Set keys in a message. Takes a map of key-value pairs and sets them in %% the message, overwriting any existing values. set(Message1, NewValuesMsg, Opts) -> - ?event({setting_keys, {msg1, Message1}, {msg2, NewValuesMsg}, {opts, Opts}}), - KeysToSet = - lists:filter( - fun(Key) -> not lists:member(Key, ?DEVICE_KEYS) end, - hb_pam:keys(NewValuesMsg, Opts) - ), - ?event({keys_to_set, {keys, KeysToSet}}), - { - ok, - maps:merge( - Message1, - maps:from_list( - lists:map( - fun(Key) -> - ?no_prod("Are we sure that the default device should " - "resolve values?"), - {Key, hb_pam:get(Key, NewValuesMsg, Opts)} - end, - KeysToSet - ) - ) - ) - }. + ?event({setting_keys, {msg1, Message1}, {msg2, NewValuesMsg}, {opts, Opts}}), + KeysToSet = + lists:filter( + fun(Key) -> not lists:member(Key, ?DEVICE_KEYS) end, + hb_pam:keys(NewValuesMsg, Opts) + ), + ?event({keys_to_set, {keys, KeysToSet}}), + { + ok, + maps:merge( + Message1, + maps:from_list( + lists:map( + fun(Key) -> + ?no_prod("Are we sure that the default device should " + "resolve values?"), + {Key, hb_pam:get(Key, NewValuesMsg, Opts)} + end, + KeysToSet + ) + ) + ) + }. %% @doc Remove a key or keys from a message. remove(Message1, #{ item := Key }) -> - remove(Message1, #{ items => [hb_pam:to_key(Key)] }); + remove(Message1, #{ items => [hb_pam:to_key(Key)] }); remove(Message1, #{ items := Keys }) -> - NormalizedKeysToRemove = lists:map(fun hb_pam:to_key/1, Keys), - { - ok, - maps:filtermap( - fun(KeyN, Val) -> - NormalizedKeyN = hb_pam:to_key(KeyN), - case lists:member(NormalizedKeyN, NormalizedKeysToRemove) of - true -> false; - false -> {true, Val} - end - end, - Message1 - ) - }. + NormalizedKeysToRemove = lists:map(fun hb_pam:to_key/1, Keys), + { + ok, + maps:filtermap( + fun(KeyN, Val) -> + NormalizedKeyN = hb_pam:to_key(KeyN), + case lists:member(NormalizedKeyN, NormalizedKeysToRemove) of + true -> false; + false -> {true, Val} + end + end, + Message1 + ) + }. %% @doc Get the public keys of a message. keys(Msg) -> - { - ok, - lists:filter( - fun(Key) -> not hb_private:is_private(Key) end, - maps:keys(Msg) - ) - }. + { + ok, + lists:filter( + fun(Key) -> not hb_private:is_private(Key) end, + maps:keys(Msg) + ) + }. %% @doc Return the value associated with the key as it exists in the message's %% underlying Erlang map. First check the public keys, then check case- %% insensitively if the key is a binary. get(Key, Msg) -> get(Key, Msg, #{ path => get }). get(Key, Msg, _Msg2) -> - ?event({getting_key, {key, Key}, {msg, Msg}}), - {ok, PublicKeys} = keys(Msg), - case lists:member(Key, PublicKeys) of - true -> {ok, maps:get(Key, Msg)}; - false when is_binary(Key) -> case_insensitive_get(Key, Msg); - false -> {error, not_found} - end. + ?event({getting_key, {key, Key}, {msg, Msg}}), + {ok, PublicKeys} = keys(Msg), + case lists:member(Key, PublicKeys) of + true -> {ok, maps:get(Key, Msg)}; + false when is_binary(Key) -> case_insensitive_get(Key, Msg); + false -> {error, not_found} + end. %% @doc Key matching should be case insensitive, following RFC-9110, so we %% implement a case-insensitive key lookup rather than delegating to %% `maps:get/2`. Encode the key to a binary if it is not already. case_insensitive_get(Key, Msg) -> - {ok, Keys} = keys(Msg), - case_insensitive_get(Key, Msg, Keys). + {ok, Keys} = keys(Msg), + case_insensitive_get(Key, Msg, Keys). case_insensitive_get(Key, Msg, Keys) when byte_size(Key) > 43 -> - do_case_insensitive_get(Key, Msg, Keys); + do_case_insensitive_get(Key, Msg, Keys); case_insensitive_get(Key, Msg, Keys) -> - do_case_insensitive_get(hb_pam:to_key(Key), Msg, Keys). + do_case_insensitive_get(hb_pam:to_key(Key), Msg, Keys). do_case_insensitive_get(_Key, _Msg, []) -> {error, not_found}; do_case_insensitive_get(Key, Msg, [CurrKey | Keys]) -> - case hb_pam:to_key(CurrKey) of - Key -> {ok, maps:get(Key, Msg)}; - _ -> do_case_insensitive_get(Key, Msg, Keys) - end. + case hb_pam:to_key(CurrKey) of + Key -> {ok, maps:get(Key, Msg)}; + _ -> do_case_insensitive_get(Key, Msg, Keys) + end. %%% Tests %%% Internal module functionality tests: get_keys_mod_test() -> - ?assertEqual([a], maps:keys(#{a => 1})). + ?assertEqual([a], maps:keys(#{a => 1})). is_private_mod_test() -> - ?assertEqual(true, hb_private:is_private(private)), - ?assertEqual(true, hb_private:is_private(<<"private">>)), - ?assertEqual(true, hb_private:is_private(<<"private.foo">>)), - ?assertEqual(false, hb_private:is_private(a)), - % Generate a long list of characters and check it does not match. - ?assertEqual(false, hb_private:is_private([ C || C <- lists:seq($a, $z) ])). + ?assertEqual(true, hb_private:is_private(private)), + ?assertEqual(true, hb_private:is_private(<<"private">>)), + ?assertEqual(true, hb_private:is_private(<<"private.foo">>)), + ?assertEqual(false, hb_private:is_private(a)), + % Generate a long list of characters and check it does not match. + ?assertEqual(false, hb_private:is_private([ C || C <- lists:seq($a, $z) ])). %%% Device functionality tests: keys_from_device_test() -> - ?assertEqual({ok, [a]}, hb_pam:resolve(#{a => 1}, keys)). + ?assertEqual({ok, [a]}, hb_pam:resolve(#{a => 1}, keys)). case_insensitive_get_test() -> - ?assertEqual({ok, 1}, case_insensitive_get(a, #{a => 1})), - ?assertEqual({ok, 1}, case_insensitive_get(<<"A">>, #{a => 1})), - ?assertEqual({ok, 1}, case_insensitive_get(<<"a">>, #{a => 1})). + ?assertEqual({ok, 1}, case_insensitive_get(a, #{a => 1})), + ?assertEqual({ok, 1}, case_insensitive_get(<<"A">>, #{a => 1})), + ?assertEqual({ok, 1}, case_insensitive_get(<<"a">>, #{a => 1})). private_keys_are_filtered_test() -> - ?assertEqual( - {ok, [a]}, - hb_pam:resolve(#{a => 1, private => 2}, keys) - ), - ?assertEqual( - {ok, [a]}, - hb_pam:resolve(#{a => 1, "priv_foo" => 4}, keys) - ). + ?assertEqual( + {ok, [a]}, + hb_pam:resolve(#{a => 1, private => 2}, keys) + ), + ?assertEqual( + {ok, [a]}, + hb_pam:resolve(#{a => 1, "priv_foo" => 4}, keys) + ). cannot_get_private_keys_test() -> - ?assertEqual( - {error, not_found}, - hb_pam:resolve(#{ a => 1, private_key => 2 }, private_key) - ). + ?assertEqual( + {error, not_found}, + hb_pam:resolve(#{ a => 1, private_key => 2 }, private_key) + ). key_from_device_test() -> - ?assertEqual({ok, 1}, hb_pam:resolve(#{a => 1}, a)). + ?assertEqual({ok, 1}, hb_pam:resolve(#{a => 1}, a)). remove_test() -> - Msg = #{ <<"Key1">> => <<"Value1">>, <<"Key2">> => <<"Value2">> }, - ?assertMatch({ok, #{ <<"Key2">> := <<"Value2">> }}, - hb_pam:resolve(Msg, #{ path => remove, item => <<"Key1">> })), - ?assertMatch({ok, #{}}, - hb_pam:resolve(Msg, #{ path => remove, items => [<<"Key1">>, <<"Key2">>] })). \ No newline at end of file + Msg = #{ <<"Key1">> => <<"Value1">>, <<"Key2">> => <<"Value2">> }, + ?assertMatch({ok, #{ <<"Key2">> := <<"Value2">> }}, + hb_pam:resolve(Msg, #{ path => remove, item => <<"Key1">> })), + ?assertMatch({ok, #{}}, + hb_pam:resolve(Msg, #{ path => remove, items => [<<"Key1">>, <<"Key2">>] })). \ No newline at end of file diff --git a/src/dev_meta.erl b/src/dev_meta.erl index 7c84cf45..8ef32374 100644 --- a/src/dev_meta.erl +++ b/src/dev_meta.erl @@ -22,21 +22,21 @@ %% @doc Execute a message on hyperbeam. execute(CarrierMsg, S) -> - ?event({executing_message, CarrierMsg}), - {Msg, Path} = parse_carrier_msg(CarrierMsg, S), - execute_path(Path, Msg, S). + ?event({executing_message, CarrierMsg}), + {Msg, Path} = parse_carrier_msg(CarrierMsg, S), + execute_path(Path, Msg, S). %% @doc Execute a path on a message, yielding a new message. execute_path([], Msg, _) -> {ok, Msg}; execute_path([FuncName|Path], Msg, S) -> - ?event({meta_executing_on_path, {function, FuncName}, {path, Path}}), - Func = parse_path_to_func(FuncName), - execute_path(Path, hb_pam:resolve(Msg, Func, [Msg], S), S). + ?event({meta_executing_on_path, {function, FuncName}, {path, Path}}), + Func = parse_path_to_func(FuncName), + execute_path(Path, hb_pam:resolve(Msg, Func, [Msg], S), S). parse_path_to_func(BinName) when is_binary(BinName) -> - binary_to_existing_atom(string:lowercase(BinName), utf8); + binary_to_existing_atom(string:lowercase(BinName), utf8); parse_path_to_func(AtomName) when is_atom(AtomName) -> - AtomName. + AtomName. %% @doc Resolve the carrier message to an executable message, either by extracting %% from its body or reading from its referenced ID. @@ -45,7 +45,7 @@ parse_carrier_msg(CarrierMsg, S) -> {_, _} -> % If the carrier message itself contains a device, we should execute % the path on that device. - ?event({carrier_msg_contains_device, CarrierMsg}), + ?event({carrier_msg_contains_device, CarrierMsg}), { path_from_carrier_message(CarrierMsg), CarrierMsg @@ -61,33 +61,33 @@ parse_carrier_msg(CarrierMsg, S) -> %% MessageWithID.Schedule(#{From => 0, To => 1}). load_executable_message_from_carrier(CarrierMsg, #{ store := RawStore }) -> Path = path_from_carrier_message(CarrierMsg), - Store = - case hb_opts:get(access_remote_cache_for_client) of - true -> RawStore; - false -> hb_store:scope(RawStore, local) - end, - load_path(Store, Path). + Store = + case hb_opts:get(access_remote_cache_for_client) of + true -> RawStore; + false -> hb_store:scope(RawStore, local) + end, + load_path(Store, Path). %% @doc Load a path from the cache. Load the whole path if possible, %% backing off to smaller parts of the path until a viable component %% (eventually just the message ID) is found. load_path(Store, PathParts) -> load_path(Store, PathParts, []). load_path(Store, PathParts, Unresolved) -> - ?event({loading_path, Store, PathParts, Unresolved}), - % First, try to read the path directly from the cache. + ?event({loading_path, Store, PathParts, Unresolved}), + % First, try to read the path directly from the cache. case hb_cache:read(Store, PathParts) of {ok, Msg} -> {Msg, Unresolved}; - not_found -> - % If that fails, try to read it as a message. - case hb_cache:read_message(Store, PathParts) of - {ok, Msg} -> {Msg, Unresolved}; - not_found -> - load_path( - Store, - lists:droplast(PathParts), - Unresolved ++ [lists:last(PathParts)] - ) - end + not_found -> + % If that fails, try to read it as a message. + case hb_cache:read_message(Store, PathParts) of + {ok, Msg} -> {Msg, Unresolved}; + not_found -> + load_path( + Store, + lists:droplast(PathParts), + Unresolved ++ [lists:last(PathParts)] + ) + end end. %% @doc Extract the components of the path from the carrier message. diff --git a/src/dev_mu.erl b/src/dev_mu.erl index 261bc072..4274f415 100644 --- a/src/dev_mu.erl +++ b/src/dev_mu.erl @@ -19,11 +19,11 @@ push(CarrierMsg, State) -> CarriedMsg; _ -> CarrierMsg end, - ?event({starting_push_for, - {unsigned, hb_util:id(Msg, unsigned)}, - {signed, hb_util:id(Msg, signed)}, - {target, hb_util:id(Msg#tx.target)} - }), + ?event({starting_push_for, + {unsigned, hb_util:id(Msg, unsigned)}, + {signed, hb_util:id(Msg, signed)}, + {target, hb_util:id(Msg#tx.target)} + }), ?no_prod(fix_mu_push_validation), case ar_bundles:verify_item(Msg) of _ -> @@ -33,22 +33,22 @@ push(CarrierMsg, State) -> X -> X end, hb_logger:register(Logger), - fork( - #result { - messages = [Msg] - }, + fork( + #result { + messages = [Msg] + }, State#{ - depth => 0, + depth => 0, store => maps:get(store, State, hb_opts:get(store)), logger => Logger, wallet => maps:get(wallet, State, hb:wallet()) } - ), - % TODO: Implement trace waiting. - ResTX = ar_bundles:sign_item( - #tx{ tags = [{<<"Status">>, <<"200">>}]}, - hb:wallet()), - {ok, #{ results => ResTX }} + ), + % TODO: Implement trace waiting. + ResTX = ar_bundles:sign_item( + #tx{ tags = [{<<"Status">>, <<"200">>}]}, + hb:wallet()), + {ok, #{ results => ResTX }} %false -> % {error, cannot_push_invalid_message} end. @@ -57,41 +57,41 @@ push(CarrierMsg, State) -> fork(Res, Opts) -> push_messages(upload, Res#result.spawns, Opts), push_messages(upload, Res#result.messages, Opts), - push_messages(attest, Res#result.assignments, Opts). + push_messages(attest, Res#result.assignments, Opts). push_messages(upload, Messages, Opts) -> lists:foreach( fun(Message) -> spawn( fun() -> - ?event( - {mu_forking_for, - {unsigned, hb_util:id(Message, unsigned)}, - {signed, hb_util:id(Message, signed)}, - {target, hb_util:id(Message#tx.target)}, - {logger, maps:get(logger, Opts, undefined)} - } - ), - Stack = dev_stack:create(?PUSH_DEV_STACK), + ?event( + {mu_forking_for, + {unsigned, hb_util:id(Message, unsigned)}, + {signed, hb_util:id(Message, signed)}, + {target, hb_util:id(Message#tx.target)}, + {logger, maps:get(logger, Opts, undefined)} + } + ), + Stack = dev_stack:create(?PUSH_DEV_STACK), {ok, Results} = hb_pam:resolve( - {dev_stack, execute}, - push, - [ - #{ - devices => Stack, - message => Message, - logger => maps:get(logger, Opts, undefined), - store => maps:get(store, Opts, hb_opts:get(store)), - wallet => maps:get(wallet, Opts, hb:wallet()) - } - ] - ), - ?event({pushing_result_for_computed_message, - {unsigned, hb_util:id(Message, unsigned)}, - {signed, hb_util:id(Message, signed)}, - {target, hb_util:id(Message#tx.target)} - }), - handle_push_result(Results, Opts) + {dev_stack, execute}, + push, + [ + #{ + devices => Stack, + message => Message, + logger => maps:get(logger, Opts, undefined), + store => maps:get(store, Opts, hb_opts:get(store)), + wallet => maps:get(wallet, Opts, hb:wallet()) + } + ] + ), + ?event({pushing_result_for_computed_message, + {unsigned, hb_util:id(Message, unsigned)}, + {signed, hb_util:id(Message, signed)}, + {target, hb_util:id(Message#tx.target)} + }), + handle_push_result(Results, Opts) end ) end, @@ -102,27 +102,27 @@ push_messages(attest, Assignments, #{ logger := Logger }) -> fun(Assignment) -> hb_logger:log(Logger, {ok, "Assigning ", ar_bundles:id(Assignment, signed)}), hb_client:assign(Assignment), - ?no_prod("After assigning, don't we want to push the message?") + ?no_prod("After assigning, don't we want to push the message?") end, maybe_to_list(Assignments) ). handle_push_result(Results, Opts = #{ depth := Depth }) -> - % Second pass: We have the results, so we can fork the messages/spawns/... - Res = #result{ - messages = maps:get(<<"/Outbox">>, Results, #{}), - assignments = maps:get(<<"/Assignment">>, Results, #{}), - spawns = maps:get(<<"/Spawn">>, Results, #{}) - }, - ?event({push_recursing, - {depth, Depth}, - {messages, maps:size(Res#result.messages)}, - {assignments, maps:size(Res#result.assignments)}, - {spawns, maps:size(Res#result.spawns)} - }), - fork(Res, Opts#{ depth => Depth + 1 }). + % Second pass: We have the results, so we can fork the messages/spawns/... + Res = #result{ + messages = maps:get(<<"/Outbox">>, Results, #{}), + assignments = maps:get(<<"/Assignment">>, Results, #{}), + spawns = maps:get(<<"/Spawn">>, Results, #{}) + }, + ?event({push_recursing, + {depth, Depth}, + {messages, maps:size(Res#result.messages)}, + {assignments, maps:size(Res#result.assignments)}, + {spawns, maps:size(Res#result.spawns)} + }), + fork(Res, Opts#{ depth => Depth + 1 }). maybe_to_list(Map) when is_map(Map) -> [V || {_K, V} <- maps:to_list(Map)]; maybe_to_list(undefined) -> []; maybe_to_list(Else) when not is_list(Else) -> [Else]; -maybe_to_list(Else) -> Else. +maybe_to_list(Else) -> Else. \ No newline at end of file diff --git a/src/dev_multipass.erl b/src/dev_multipass.erl index c2fc2e71..f58736a6 100644 --- a/src/dev_multipass.erl +++ b/src/dev_multipass.erl @@ -14,4 +14,4 @@ execute(_, S = #{ pass := Pass, passes := Passes }) when Pass < Passes -> execute(_, S) -> {ok, S}. -uses() -> all. +uses() -> all. \ No newline at end of file diff --git a/src/dev_p4.erl b/src/dev_p4.erl index 23801a2d..f94088a3 100644 --- a/src/dev_p4.erl +++ b/src/dev_p4.erl @@ -3,4 +3,4 @@ push(_Item, S) -> % TODO: Check payment. - {ok, S}. + {ok, S}. \ No newline at end of file diff --git a/src/dev_poda.erl b/src/dev_poda.erl index 0c0c8fca..d6c6e562 100644 --- a/src/dev_poda.erl +++ b/src/dev_poda.erl @@ -24,8 +24,8 @@ extract_opts(Params) -> Authorities = lists:filtermap( fun({<<"Authority">>, Addr}) -> {true, Addr}; - (_) -> false end, - Params + (_) -> false end, + Params ), {_, RawQuorum} = lists:keyfind(<<"Quorum">>, 1, Params), Quorum = binary_to_integer(RawQuorum), @@ -139,18 +139,18 @@ validate_attestation(Msg, Att, Opts) -> (lists:keyfind(<<"Attestation-For">>, 1, Att#tx.tags) == {<<"Attestation-For">>, MsgID}) orelse ar_bundles:member(ar_bundles:id(Msg, unsigned), Att), - case ValidSigner and ValidSignature and RelevantMsg of - false -> - ?event({poda_attestation_invalid, - {attestation, ar_bundles:id(Att, signed)}, - {signer, AttSigner}, - {valid_signer, ValidSigner}, - {valid_signature, ValidSignature}, - {relevant_msg, RelevantMsg}} - ), - false; - true -> true - end. + case ValidSigner and ValidSignature and RelevantMsg of + false -> + ?event({poda_attestation_invalid, + {attestation, ar_bundles:id(Att, signed)}, + {signer, AttSigner}, + {valid_signer, ValidSigner}, + {valid_signature, ValidSignature}, + {relevant_msg, RelevantMsg}} + ), + false; + true -> true + end. %%% Execution flow: Error handling. %%% Skip execution of this message, instead returning an error message. @@ -189,8 +189,8 @@ attest_to_results(Msg, S) -> % Add attestations to the outbox and spawn items. maps:map( fun(Key, IndexMsg) -> - ?no_prod("Currently we only attest to the outbox and spawn items." - "Make it general?"), + ?no_prod("Currently we only attest to the outbox and spawn items." + "Make it general?"), case lists:member(Key, [<<"/Outbox">>, <<"/Spawn">>]) of true -> ?event({poda_starting_to_attest_to_result, Key}), @@ -216,7 +216,7 @@ add_attestations(NewMsg, S = #{ assignment := Assignment, store := _Store, logge % Aggregate validations from other nodes. % TODO: Filter out attestations from the current node. MsgID = hb_util:encode(ar_bundles:id(NewMsg, unsigned)), - ?event({poda_add_attestations_from, InitAuthorities, {self,hb:address()}}), + ?event({poda_add_attestations_from, InitAuthorities, {self,hb:address()}}), Attestations = pfiltermap( fun(Address) -> case hb_router:find(compute, ar_bundles:id(Process, unsigned), Address) of diff --git a/src/dev_scheduler.erl b/src/dev_scheduler.erl index 179f4f06..ee35d6da 100644 --- a/src/dev_scheduler.erl +++ b/src/dev_scheduler.erl @@ -18,7 +18,7 @@ %%% HTTP API functions: schedule(Item) -> {ok, Output} = dev_scheduler_interface:handle(Item), - %?debug_wait(1000), + %?debug_wait(1000), {ok, Output}. %%% MU pushing client functions: @@ -26,10 +26,10 @@ push(Msg, State = #{ logger := Logger }) -> ?event(su_scheduling_message_for_push), case hb_client:schedule(Msg) of {ok, Assignment} -> - ?event({scheduled_message, hb_util:id(Assignment, unsigned)}), + ?event({scheduled_message, hb_util:id(Assignment, unsigned)}), {ok, State#{assignment => Assignment}}; Error -> - ?event({error_scheduling_message, Error}), + ?event({error_scheduling_message, Error}), hb_logger:log(Logger, Error), {error, Error} end. @@ -54,13 +54,13 @@ update_schedule(State = #{ process := Proc }) -> % TODO: Get from slot via checkpoint. (Done, right?) Assignments = hb_client:get_assignments(hb_util:id(Proc, signed), CurrentSlot, ToSlot), ?event({got_assignments_from_su, - [ - { - element(2, lists:keyfind(<<"Assignment">>, 1, A#tx.tags)), - hb_util:id(A, signed), - hb_util:id(A, unsigned) - } - || A <- Assignments ]}), + [ + { + element(2, lists:keyfind(<<"Assignment">>, 1, A#tx.tags)), + hb_util:id(A, signed), + hb_util:id(A, unsigned) + } + || A <- Assignments ]}), lists:foreach( fun(Assignment) -> ?event({writing_assignment_to_cache, hb_util:id(Assignment, unsigned)}), diff --git a/src/dev_scheduler_interface.erl b/src/dev_scheduler_interface.erl index e07b9107..f4c2685b 100644 --- a/src/dev_scheduler_interface.erl +++ b/src/dev_scheduler_interface.erl @@ -65,9 +65,9 @@ current_schedule(M) -> schedule(CarrierM) -> ?event(scheduling_message), #{ <<"1">> := M } = CarrierM#tx.data, - %ar_bundles:print(M), + %ar_bundles:print(M), Store = hb_opts:get(store), - ?no_prod("SU does not validate item before writing into stream."), + ?no_prod("SU does not validate item before writing into stream."), case {ar_bundles:verify_item(M), lists:keyfind(<<"Type">>, 1, M#tx.tags)} of % {false, _} -> % {ok, @@ -114,7 +114,7 @@ send_schedule(Store, ProcID, From, {<<"To">>, To}) -> send_schedule(Store, ProcID, From, binary_to_integer(To)); send_schedule(Store, ProcID, From, To) -> {Timestamp, Height, Hash} = ar_timestamp:get(), - ?event({servicing_request_for_assignments, {proc_id, ProcID}, {from, From}, {to, To}}), + ?event({servicing_request_for_assignments, {proc_id, ProcID}, {from, From}, {to, To}}), {Assignments, More} = dev_scheduler_server:get_assignments( ProcID, From, @@ -160,7 +160,7 @@ assignments_to_bundle(Store, [Assignment | Assignments], Bundle) -> {_, Slot} = lists:keyfind(<<"Slot">>, 1, Assignment#tx.tags), {_, MessageID} = lists:keyfind(<<"Message">>, 1, Assignment#tx.tags), {ok, Message} = hb_cache:read_message(Store, MessageID), - ?event({adding_assignment_to_bundle, Slot, {requested, MessageID}, hb_util:id(Assignment, signed), hb_util:id(Assignment, unsigned)}), + ?event({adding_assignment_to_bundle, Slot, {requested, MessageID}, hb_util:id(Assignment, signed), hb_util:id(Assignment, unsigned)}), assignments_to_bundle( Store, Assignments, diff --git a/src/dev_scheduler_registry.erl b/src/dev_scheduler_registry.erl index db781a21..bf602970 100644 --- a/src/dev_scheduler_registry.erl +++ b/src/dev_scheduler_registry.erl @@ -36,7 +36,7 @@ maybe_new_proc(_ProcID, false) -> not_found; maybe_new_proc(ProcID, GenIfNotHosted) when is_binary(ProcID) -> maybe_new_proc(binary_to_list(ProcID), GenIfNotHosted); maybe_new_proc(ProcID, _) -> - ?event({starting_su_for, ProcID}), + ?event({starting_su_for, ProcID}), Pid = dev_scheduler_server:start(ProcID, get_wallet()), try pg:join({su, ProcID}, Pid), @@ -81,6 +81,6 @@ get_all_processes_test() -> ?MODULE:find(?TEST_PROC_ID2, true), Processes = ?MODULE:get_processes(), ?assertEqual(2, length(Processes)), - ?event({processes, Processes}), + ?event({processes, Processes}), ?assert(lists:member(?TEST_PROC_ID1, Processes)), - ?assert(lists:member(?TEST_PROC_ID2, Processes)). + ?assert(lists:member(?TEST_PROC_ID2, Processes)). \ No newline at end of file diff --git a/src/dev_scheduler_server.erl b/src/dev_scheduler_server.erl index a51375cf..f569e461 100644 --- a/src/dev_scheduler_server.erl +++ b/src/dev_scheduler_server.erl @@ -134,11 +134,11 @@ do_assign(State, Message, ReplyPID) -> {<<"Process">>, hb_util:id(State#state.id)}, {<<"Epoch">>, <<"0">>}, {<<"Slot">>, list_to_binary(integer_to_list(NextNonce))}, - % This was causing an error during tag encoding, - % due to badarg on byte_length. Not sure that accessing - % Message as a record (like process id from State above) - % is the correct solution. - {<<"Message">>, hb_util:id(Message, signed)}, + % This was causing an error during tag encoding, + % due to badarg on byte_length. Not sure that accessing + % Message as a record (like process id from State above) + % is the correct solution. + {<<"Message">>, hb_util:id(Message, signed)}, {<<"Block-Height">>, list_to_binary(integer_to_list(Height))}, {<<"Block-Hash">>, Hash}, {<<"Block-Timestamp">>, list_to_binary(integer_to_list(Timestamp))}, @@ -187,4 +187,4 @@ new_proc_test() -> schedule(ID, SignedItem2), schedule(ID, SignedItem3), ?assertEqual(2, dev_scheduler_server:get_current_slot( - dev_scheduler_registry:find(ID))). \ No newline at end of file + dev_scheduler_registry:find(ID))). \ No newline at end of file diff --git a/src/dev_stack.erl b/src/dev_stack.erl index e57f4303..dd7e61de 100644 --- a/src/dev_stack.erl +++ b/src/dev_stack.erl @@ -90,109 +90,109 @@ info() -> %% except for `set/2` which is handled by the default implementation in %% `dev_message`. router(set, Message1, Message2, Opts) -> - dev_message:set(Message1, Message2, Opts); + dev_message:set(Message1, Message2, Opts); router(Key, Message1, Message2, Opts) -> - InitDevMsg = hb_pam:get(<<"Device">>, Message1, Opts), - case resolve_stack(Key, Message1, Message2, Opts) of - {ok, Result} when is_map(Result) -> - {ok, hb_pam:set(Result, <<"Device">>, InitDevMsg, Opts)}; - Else -> Else - end. + InitDevMsg = hb_pam:get(<<"Device">>, Message1, Opts), + case resolve_stack(Key, Message1, Message2, Opts) of + {ok, Result} when is_map(Result) -> + {ok, hb_pam:set(Result, <<"Device">>, InitDevMsg, Opts)}; + Else -> Else + end. %% @doc Return Message1, transformed such that the device named `Key` from the %% `Device-Stack` key in the message takes the place of the original `Device` %% key. This transformation allows dev_stack to correctly track the HashPath %% of the message as it delegates execution to devices contained within it. transform_device(Message1, Key, Opts) -> - case hb_pam:resolve(Message1, #{ path => [<<"Device-Stack">>, Key] }, Opts) of - {ok, DevMsg} -> - {ok, - hb_pam:set( - Message1, - #{ <<"Device">> => DevMsg }, - Opts - ) - }; - _ -> not_found - end. + case hb_pam:resolve(Message1, #{ path => [<<"Device-Stack">>, Key] }, Opts) of + {ok, DevMsg} -> + {ok, + hb_pam:set( + Message1, + #{ <<"Device">> => DevMsg }, + Opts + ) + }; + _ -> not_found + end. %% @doc The main device stack execution engine. See the `moduledoc` for more %% information. resolve_stack(Message1, Key, Message2, Opts) -> - resolve_stack(Message1, Key, Message2, 1, Opts). + resolve_stack(Message1, Key, Message2, 1, Opts). resolve_stack(Message1, Key, Message2, DevNum, Opts) -> - case transform_device(Message1, integer_to_binary(DevNum), Opts) of - {ok, Message3} -> - ?event({stack_executing_device, DevNum, Message3}), - case hb_pam:resolve(Message3, Message2, Opts) of - {ok, Message4} when is_map(Message4) -> - resolve_stack(Message4, Key, Message2, DevNum + 1, Opts); - {skip, Message4} when is_map(Message4) -> - {ok, Message4}; - {pass, Message4} when is_map(Message4) -> - case hb_pam:resolve(Message4, pass, Opts) of - {ok, <<"Allow">>} -> - ?event({stack_repassing, DevNum, Message4}), - resolve_stack( - hb_pam:set(Message4, pass, 1, Opts), - Key, - Message2, - 1, - Opts - ); - _ -> - maybe_error( - Message1, - Key, - Message2, - DevNum + 1, - Opts, - {pass_not_allowed, Message4} - ) - end; - {error, Info} -> - maybe_error(Message1, Key, Message2, DevNum + 1, Opts, Info); - Unexpected -> - maybe_error( - Message1, - Key, - Message2, - DevNum + 1, - Opts, - {unexpected_pam_result, Unexpected} - ) - end; - not_found -> - ?event({stack_execution_finished, DevNum, Message1}), - {ok, Message1} - end. + case transform_device(Message1, integer_to_binary(DevNum), Opts) of + {ok, Message3} -> + ?event({stack_executing_device, DevNum, Message3}), + case hb_pam:resolve(Message3, Message2, Opts) of + {ok, Message4} when is_map(Message4) -> + resolve_stack(Message4, Key, Message2, DevNum + 1, Opts); + {skip, Message4} when is_map(Message4) -> + {ok, Message4}; + {pass, Message4} when is_map(Message4) -> + case hb_pam:resolve(Message4, pass, Opts) of + {ok, <<"Allow">>} -> + ?event({stack_repassing, DevNum, Message4}), + resolve_stack( + hb_pam:set(Message4, pass, 1, Opts), + Key, + Message2, + 1, + Opts + ); + _ -> + maybe_error( + Message1, + Key, + Message2, + DevNum + 1, + Opts, + {pass_not_allowed, Message4} + ) + end; + {error, Info} -> + maybe_error(Message1, Key, Message2, DevNum + 1, Opts, Info); + Unexpected -> + maybe_error( + Message1, + Key, + Message2, + DevNum + 1, + Opts, + {unexpected_pam_result, Unexpected} + ) + end; + not_found -> + ?event({stack_execution_finished, DevNum, Message1}), + {ok, Message1} + end. maybe_error(Message1, Key, Message2, DevNum, Info, Opts) -> case hb_pam:get(<<"Error-Strategy">>, Message1, Opts) of <<"Stop">> -> - {error, {stack_call_failed, Message1, Key, Message2, DevNum, Info}}; + {error, {stack_call_failed, Message1, Key, Message2, DevNum, Info}}; <<"Throw">> -> - throw({error_running_dev, Message1, Key, Message2, DevNum, Info}); + throw({error_running_dev, Message1, Key, Message2, DevNum, Info}); <<"Continue">> -> - ?event({continue_stack_execution_after_error, Message1, Key, Info}), + ?event({continue_stack_execution_after_error, Message1, Key, Info}), resolve_stack( hb_pam:set(Message1, - [ - <<"Errors">>, - hb_pam:get(id, Message1, Opts), - hb_pam:get(pass, Message1, Opts), - DevNum, - hb_util:debug_fmt(Info) - ], - Opts - ), + [ + <<"Errors">>, + hb_pam:get(id, Message1, Opts), + hb_pam:get(pass, Message1, Opts), + DevNum, + hb_util:debug_fmt(Info) + ], + Opts + ), Key, - Message2, + Message2, DevNum + 1, Opts ); <<"Ignore">> -> - ?event({ignoring_stack_error, Message1, Key, Info}), + ?event({ignoring_stack_error, Message1, Key, Info}), resolve_stack( Message1, Key, @@ -205,12 +205,12 @@ maybe_error(Message1, Key, Message2, DevNum, Info, Opts) -> %%% Tests generate_append_device(Str) -> - #{ - test => - fun(#{ bin := Bin }) -> - {ok, << Bin/binary, Str/bitstring >>} - end - }. + #{ + test => + fun(#{ bin := Bin }) -> + {ok, << Bin/binary, Str/bitstring >>} + end + }. %% Create a device that modifies a number when the user tries to set it to a %% new value. This is a 'wonky' set device because it does not actually `set`, @@ -219,46 +219,46 @@ generate_append_device(Str) -> %% increasing the likelihood that a subtle error would be exposed by these %% tests. generate_wonky_set_device(Modifier) -> - #{ - set => - fun(Msg1, Msg2) -> - % Find the first key that is not a path. - Key = hd(maps:keys(Msg2) -- [path, hashpath]), - % Set it not to the new value, but the current value plus the - % modifier. - {ok, hb_pam:set(Msg1, Key, hb_pam:get(Key, Msg2) + Modifier)} - end - }. + #{ + set => + fun(Msg1, Msg2) -> + % Find the first key that is not a path. + Key = hd(maps:keys(Msg2) -- [path, hashpath]), + % Set it not to the new value, but the current value plus the + % modifier. + {ok, hb_pam:set(Msg1, Key, hb_pam:get(Key, Msg2) + Modifier)} + end + }. transform_device_test_ignore() -> - WonkyDev = generate_wonky_set_device(1), - Msg1 = - #{ - <<"Device">> => <<"Stack">>, - <<"Device-Stack">> => - #{ - <<"1">> => WonkyDev, - <<"2">> => <<"Message">> - } - }, - ?assertEqual( - {ok, #{ <<"Device">> => <<"Message">> } }, - transform_device(Msg1, <<"2">>, #{}) - ). + WonkyDev = generate_wonky_set_device(1), + Msg1 = + #{ + <<"Device">> => <<"Stack">>, + <<"Device-Stack">> => + #{ + <<"1">> => WonkyDev, + <<"2">> => <<"Message">> + } + }, + ?assertEqual( + {ok, #{ <<"Device">> => <<"Message">> } }, + transform_device(Msg1, <<"2">>, #{}) + ). simple_stack_execute_test_ignore() -> - Msg = #{ - <<"Device">> => ?MODULE, - <<"Device-Stack">> => - #{ - <<"1">> => generate_append_device("1"), - <<"2">> => generate_append_device("2") - } - }, - ?assertEqual( - {ok, #{ bin => <<"12">> }}, - hb_pam:resolve(Msg, test) - ). + Msg = #{ + <<"Device">> => ?MODULE, + <<"Device-Stack">> => + #{ + <<"1">> => generate_append_device("1"), + <<"2">> => generate_append_device("2") + } + }, + ?assertEqual( + {ok, #{ bin => <<"12">> }}, + hb_pam:resolve(Msg, test) + ). %% Ensure that devices are reordered correctly during execution. We use the %% 'wonky set' device defined above, as well as the default `message` device, @@ -281,18 +281,18 @@ simple_stack_execute_test_ignore() -> %% { x => 10 } %% Msg2.x => 10 stack_reorder_test_ignore() -> - WonkyDev = generate_wonky_set_device(1), - Msg1 = - #{ - <<"Device">> => <<"Stack">>, - <<"Device-Stack">> => - #{ - <<"1">> => <<"Message">>, - <<"2">> => WonkyDev - }, - x => 1 - }, - ?assertEqual( - {ok, #{ x => 6 }}, - hb_pam:resolve(Msg1, #{ path => [set], x => 5 }) - ). \ No newline at end of file + WonkyDev = generate_wonky_set_device(1), + Msg1 = + #{ + <<"Device">> => <<"Stack">>, + <<"Device-Stack">> => + #{ + <<"1">> => <<"Message">>, + <<"2">> => WonkyDev + }, + x => 1 + }, + ?assertEqual( + {ok, #{ x => 6 }}, + hb_pam:resolve(Msg1, #{ path => [set], x => 5 }) + ). \ No newline at end of file diff --git a/src/dev_vfs.erl b/src/dev_vfs.erl index b3a25cdd..f3cad891 100644 --- a/src/dev_vfs.erl +++ b/src/dev_vfs.erl @@ -76,7 +76,7 @@ execute(_M, S) -> %% @doc Return the stdout buffer from a state message. stdout(#{ vfs := #{ 1 := #fd { data = Data } } }) -> - Data. + Data. path_open(S = #{ vfs := FDs }, Port, [FDPtr, LookupFlag, PathPtr|_]) -> ?event({path_open, FDPtr, LookupFlag, PathPtr}), diff --git a/src/dev_wasm.erl b/src/dev_wasm.erl index 0086d77e..537d3e21 100644 --- a/src/dev_wasm.erl +++ b/src/dev_wasm.erl @@ -65,4 +65,4 @@ terminate(State = #{wasm := Port}) -> hb_beamr:stop(Port), {ok, State#{wasm := undefined}}. -uses() -> all. +uses() -> all. \ No newline at end of file diff --git a/src/hb.app.src b/src/hb.app.src index 8c1dd5ee..bf2fb0eb 100644 --- a/src/hb.app.src +++ b/src/hb.app.src @@ -1,21 +1,21 @@ {application, hb, [ - {description, "HyperBEAM: An Erlang implementation of a permaweb supercomputer."}, - {vsn, "0.0.1"}, - {registered, []}, - {mod, {hb_app, []}}, - {applications, [ - kernel, - stdlib, - inets, - ssl, - debugger, - cowboy, - prometheus, - prometheus_cowboy, - os_mon - ]}, - {env, []}, - {modules, []}, - {licenses, ["BSL-1.1"]}, - {links, []} -]}. + {description, "HyperBEAM: An Erlang implementation of a permaweb supercomputer."}, + {vsn, "0.0.1"}, + {registered, []}, + {mod, {hb_app, []}}, + {applications, [ + kernel, + stdlib, + inets, + ssl, + debugger, + cowboy, + prometheus, + prometheus_cowboy, + os_mon + ]}, + {env, []}, + {modules, []}, + {licenses, ["BSL-1.1"]}, + {links, []} +]}. \ No newline at end of file diff --git a/src/hb.erl b/src/hb.erl index 5effd699..ef19e29e 100644 --- a/src/hb.erl +++ b/src/hb.erl @@ -90,21 +90,21 @@ %% @doc Initialize system-wide settings for the hyperbeam node. init() -> - ?event({setting_debug_stack_depth, hb_opts:get(debug_stack_depth)}), + ?event({setting_debug_stack_depth, hb_opts:get(debug_stack_depth)}), Old = erlang:system_flag(backtrace_depth, hb_opts:get(debug_stack_depth)), - ?event({old_system_stack_depth, Old}), - ok. + ?event({old_system_stack_depth, Old}), + ok. wallet() -> wallet(hb_opts:get(key_location)). wallet(Location) -> case file:read_file_info(Location) of {ok, _} -> - ar_wallet:load_keyfile(Location); + ar_wallet:load_keyfile(Location); {error, _} -> - Res = ar_wallet:new_keyfile(?DEFAULT_KEY_TYPE, Location), - ?event({created_new_keyfile, Location, address(Res)}), - Res + Res = ar_wallet:new_keyfile(?DEFAULT_KEY_TYPE, Location), + ?event({created_new_keyfile, Location, address(Res)}), + Res end. %% @doc Get the address of a wallet. Defaults to the address of the wallet @@ -133,9 +133,9 @@ event(X, ModAtom, Func, Line) when is_atom(ModAtom) -> end; event(X, ModStr, Func, Line) -> case hb_opts:get(debug_print) of - ModList when is_list(ModList) -> - lists:member(ModStr, ModList) andalso - hb_util:debug_print(X, ModStr, Func, Line); + ModList when is_list(ModList) -> + lists:member(ModStr, ModList) andalso + hb_util:debug_print(X, ModStr, Func, Line); true -> hb_util:debug_print(X, ModStr, Func, Line); false -> X end. @@ -158,10 +158,10 @@ no_prod(X, Mod, Line) -> io:format(standard_error, "=== DANGER: NON-PROD READY CODE INVOKED IN PROD ===~n", []), io:format(standard_error, "~w:~w: ~p~n", [Mod, Line, X]), - case hb_opts:get(exit_on_no_prod) of - true -> init:stop(); - false -> throw(X) - end; + case hb_opts:get(exit_on_no_prod) of + true -> init:stop(); + false -> throw(X) + end; _ -> X end. @@ -188,6 +188,6 @@ profile(Fun) -> %% message to the console first. debug_wait(T, Mod, Func, Line) -> hb_util:debug_print( - lists:flatten(io_lib:format("[Debug waiting ~pms...]", [T])), - Mod, Func, Line), - receive after T -> ok end. + lists:flatten(io_lib:format("[Debug waiting ~pms...]", [T])), + Mod, Func, Line), + receive after T -> ok end. \ No newline at end of file diff --git a/src/hb_app.erl b/src/hb_app.erl index 4d7255c5..9bbe479c 100644 --- a/src/hb_app.erl +++ b/src/hb_app.erl @@ -19,4 +19,4 @@ start(_StartType, _StartArgs) -> {ok, _} = hb_http_router:start(). stop(_State) -> - ok. + ok. \ No newline at end of file diff --git a/src/hb_beamr.erl b/src/hb_beamr.erl index 7dc737a5..45b31ae3 100644 --- a/src/hb_beamr.erl +++ b/src/hb_beamr.erl @@ -101,13 +101,13 @@ simple_wasm_test() -> ?assertEqual(120.0, Result). generate_test_vfs() -> - dev_vfs:init(#{}). + dev_vfs:init(#{}). test_call(Port, Func, Args) -> - {ok, VFSState0} = generate_test_vfs(), - test_call(VFSState0, Port, Func, Args). + {ok, VFSState0} = generate_test_vfs(), + test_call(VFSState0, Port, Func, Args). test_call(VFSState, Port, Func, Args) -> - call(VFSState, Port, Func, Args, fun dev_json_iface:stdlib/6). + call(VFSState, Port, Func, Args, fun dev_json_iface:stdlib/6). simple_wasm_calling_test() -> {ok, File} = file:read_file("test/test-calling.wasm"), @@ -120,9 +120,9 @@ simple_wasm_calling_test() -> {ok, Ptr1} = hb_beamr_io:write_string(Port, Arg1), ?assertNotEqual(0, Ptr1), {ok, [], State} = test_call(Port, "print_args", [Ptr0, Ptr1]), - Str = dev_vfs:stdout(State), - ?assert(binary:match(Str, <<"Test string arg 00000000000000">>) /= nomatch), - ?assert(binary:match(Str, <<"Test string arg 11111111111111">>) /= nomatch). + Str = dev_vfs:stdout(State), + ?assert(binary:match(Str, <<"Test string arg 00000000000000">>) /= nomatch), + ?assert(binary:match(Str, <<"Test string arg 11111111111111">>) /= nomatch). wasm64_test() -> ?event(simple_wasm64_test), @@ -133,14 +133,14 @@ wasm64_test() -> gen_test_env() -> <<"{\"Process\":{\"Id\":\"AOS\",\"Owner\":\"FOOBAR\",\"Tags\":[" - "{\"name\":\"Name\",\"value\":\"Thomas\"}," - "{\"name\":\"Authority\",\"value\":\"FOOBAR\"}]}}\0">>. + "{\"name\":\"Name\",\"value\":\"Thomas\"}," + "{\"name\":\"Authority\",\"value\":\"FOOBAR\"}]}}\0">>. gen_test_aos_msg(Command) -> <<"{\"From\":\"FOOBAR\",\"Block-Height\":\"1\",\"Target\":\"AOS\"," - "\"Owner\":\"FOOBAR\",\"Id\":\"1\",\"Module\":\"W\"," - "\"Tags\":[{\"name\":\"Action\",\"value\":\"Eval\"}],\"Data\":\"", - (list_to_binary(Command))/binary, "\"}\0">>. + "\"Owner\":\"FOOBAR\",\"Id\":\"1\",\"Module\":\"W\"," + "\"Tags\":[{\"name\":\"Action\",\"value\":\"Eval\"}],\"Data\":\"", + (list_to_binary(Command))/binary, "\"}\0">>. aos64_standalone_wex_test() -> Msg = gen_test_aos_msg("return 1+1"), diff --git a/src/hb_cache.erl b/src/hb_cache.erl index 9a5cbb33..f0051088 100644 --- a/src/hb_cache.erl +++ b/src/hb_cache.erl @@ -94,7 +94,7 @@ first_slot_with_path(Store, ProcID, [LatestSlot | Rest], Limit, Path) -> build_path(PathList, Map) -> lists:map( fun(Ref) when is_atom(Ref) -> maps:get(Ref, Map); - (Other) -> Other + (Other) -> Other end, PathList ). @@ -154,13 +154,13 @@ write_assignment(Store, Assignment) -> fmt_id(ProcID), binary_to_list(Slot) ]), - AssignmentPathByID = + AssignmentPathByID = hb_store:path(Store, [ "assignments", fmt_id(ProcID), SignedID ]), - AssignmentPathByUnsignedID = + AssignmentPathByUnsignedID = hb_store:path(Store, [ "assignments", fmt_id(ProcID), @@ -311,18 +311,18 @@ read(Store, RawPath) -> %% separate the bundle items from the main object in storage, but if ar_bundles %% exposed a function to rebuild the TX object's manifest we could avoid this. {ok, - ar_bundles:deserialize(ar_bundles:normalize(Root#tx { - data = maps:map( - fun(_, Key) -> - {ok, Child} = read(Store, ["messages", fmt_id(Key)]), - Child - end, - ar_bundles:parse_manifest( - maps:get(<<"manifest">>, Root#tx.data) - ) - ) - })) - }; + ar_bundles:deserialize(ar_bundles:normalize(Root#tx { + data = maps:map( + fun(_, Key) -> + {ok, Child} = read(Store, ["messages", fmt_id(Key)]), + Child + end, + ar_bundles:parse_manifest( + maps:get(<<"manifest">>, Root#tx.data) + ) + ) + })) + }; simple ->read_simple_message(Store, MessagePath); not_found -> not_found end. diff --git a/src/hb_client.erl b/src/hb_client.erl index 687fa463..9fb46a7e 100644 --- a/src/hb_client.erl +++ b/src/hb_client.erl @@ -141,30 +141,30 @@ extract_assignments(From, To, Assignments) -> %% @doc Compute the result of a message on a process. compute(ProcIDBarer, Msg) when is_record(ProcIDBarer, tx) -> - case lists:keyfind(<<"Process">>, 1, ProcIDBarer#tx.tags) of - {<<"Process">>, ProcID} -> - % Potential point of confusion: If the ProcIDBarer message - % has a `Process` tag, then it is an _assignment_ -- not a - % process message. We extract the `Process` tag to use as the - % process ID. - {ok, Node} = hb_router:find(compute, ProcID), - compute(Node, ProcIDBarer, Msg); - false -> - case lists:keyfind(<<"Type">>, 1, ProcIDBarer#tx.tags) of - {<<"Type">>, <<"Process">>} -> - % If the ProcIDBarer message has a `Type` tag, then it is - % a _process message_ -- not an assignment. Extract its ID and - % run on that. - {ok, Node} = - hb_router:find(compute, ProcID = hb_util:id(Msg, signed)), - compute(Node, ProcID, Msg); - false -> - throw( - {unrecognized_message_to_compute_on, - hb_util:id(Msg, unsigned)} - ) - end - end. + case lists:keyfind(<<"Process">>, 1, ProcIDBarer#tx.tags) of + {<<"Process">>, ProcID} -> + % Potential point of confusion: If the ProcIDBarer message + % has a `Process` tag, then it is an _assignment_ -- not a + % process message. We extract the `Process` tag to use as the + % process ID. + {ok, Node} = hb_router:find(compute, ProcID), + compute(Node, ProcIDBarer, Msg); + false -> + case lists:keyfind(<<"Type">>, 1, ProcIDBarer#tx.tags) of + {<<"Type">>, <<"Process">>} -> + % If the ProcIDBarer message has a `Type` tag, then it is + % a _process message_ -- not an assignment. Extract its ID and + % run on that. + {ok, Node} = + hb_router:find(compute, ProcID = hb_util:id(Msg, signed)), + compute(Node, ProcID, Msg); + false -> + throw( + {unrecognized_message_to_compute_on, + hb_util:id(Msg, unsigned)} + ) + end + end. compute(Node, ProcID, Slot) -> compute(Node, ProcID, Slot, #{}). compute(Node, ProcID, Slot, Opts) when is_binary(ProcID) -> @@ -320,4 +320,4 @@ json_struct_to_result(Struct, Res) -> cursor = hb_util:find_value(<<"cursor">>, Struct, undefined) } ) - end. + end. \ No newline at end of file diff --git a/src/hb_crypto.erl b/src/hb_crypto.erl index 19773084..d074dee3 100644 --- a/src/hb_crypto.erl +++ b/src/hb_crypto.erl @@ -20,46 +20,46 @@ %% @doc Add a new ID to the end of a SHA-256 hash chain. sha256_chain(ID1, ID2) when ?IS_ID(ID1) and ?IS_ID(ID2) -> - ?no_prod("CAUTION: Unaudited cryptographic function invoked."), - sha256(<>); + ?no_prod("CAUTION: Unaudited cryptographic function invoked."), + sha256(<>); sha256_chain(ID1, ID2) -> - throw({cannot_chain_bad_ids, ID1, ID2}). + throw({cannot_chain_bad_ids, ID1, ID2}). %% @doc Accumulate two IDs into a single commitment. %% Experimental! This is not necessarily a cryptographically-secure operation. accumulate(ID1 = << ID1Int:256 >>, ID2) when ?IS_ID(ID1) and ?IS_ID(ID2) -> - ?no_prod("CAUTION: Experimental cryptographic algorithm invoked."), - << ID2Int:256 >> = sha256_chain(ID1, ID2), - << (ID1Int + ID2Int):256 >>; + ?no_prod("CAUTION: Experimental cryptographic algorithm invoked."), + << ID2Int:256 >> = sha256_chain(ID1, ID2), + << (ID1Int + ID2Int):256 >>; accumulate(ID1, ID2) -> - throw({cannot_accumulate_bad_ids, ID1, ID2}). + throw({cannot_accumulate_bad_ids, ID1, ID2}). %% @doc Wrap Erlang's `crypto:hash/2` to provide a standard interface. %% Under-the-hood, this uses OpenSSL. sha256(Data) -> - crypto:hash(sha256, Data). + crypto:hash(sha256, Data). %%% Tests %% @doc Count the number of leading zeroes in a bitstring. count_zeroes(<<>>) -> - 0; + 0; count_zeroes(<<0:1, Rest/bitstring>>) -> - 1 + count_zeroes(Rest); + 1 + count_zeroes(Rest); count_zeroes(<<_:1, Rest/bitstring>>) -> - count_zeroes(Rest). + count_zeroes(Rest). %% @doc Check that `sha-256-chain` correctly produces a hash matching %% the machine's OpenSSL lib's output. Further (in case of a bug in our %% or Erlang's usage of OpenSSL), check that the output has at least has %% a high level of entropy. sha256_chain_test() -> - ID1 = <<1:256>>, - ID2 = <<2:256>>, - ID3 = sha256_chain(ID1, ID2), - HashBase = << ID1/binary, ID2/binary >>, - ?assertEqual(ID3, crypto:hash(sha256, HashBase)), - % Basic entropy check. - Avg = count_zeroes(ID3) / 256, - ?assert(Avg > 0.4), - ?assert(Avg < 0.6). + ID1 = <<1:256>>, + ID2 = <<2:256>>, + ID3 = sha256_chain(ID1, ID2), + HashBase = << ID1/binary, ID2/binary >>, + ?assertEqual(ID3, crypto:hash(sha256, HashBase)), + % Basic entropy check. + Avg = count_zeroes(ID3) / 256, + ?assert(Avg > 0.4), + ?assert(Avg < 0.6). \ No newline at end of file diff --git a/src/hb_http.erl b/src/hb_http.erl index dd2be47a..cd143fbd 100644 --- a/src/hb_http.erl +++ b/src/hb_http.erl @@ -19,10 +19,10 @@ start() -> %% form. get(Host, Path) -> ?MODULE:get(Host ++ Path). get(URL) -> - case get_binary(URL) of - {ok, Res} -> {ok, ar_bundles:deserialize(Res)}; - Error -> Error - end. + case get_binary(URL) of + {ok, Res} -> {ok, ar_bundles:deserialize(Res)}; + Error -> Error + end. %% @doc Gets a URL via HTTP and returns the raw binary body. Abstracted such that %% we can easily swap out the HTTP client library later. @@ -42,12 +42,12 @@ get_binary(URL) -> post(Host, Path, Message) -> post(Host ++ Path, Message). post(URL, Message) when not is_binary(Message) -> ?event({http_post, hb_util:id(Message, unsigned), hb_util:id(Message, signed), URL}), - post(URL, ar_bundles:serialize(ar_bundles:normalize(Message))); + post(URL, ar_bundles:serialize(ar_bundles:normalize(Message))); post(URL, Message) -> - case post_binary(URL, Message) of - {ok, Res} -> {ok, ar_bundles:deserialize(Res)}; - Error -> Error - end. + case post_binary(URL, Message) of + {ok, Res} -> {ok, ar_bundles:deserialize(Res)}; + Error -> Error + end. %% @doc Posts a binary to a URL on a remote peer via HTTP, returning the raw %% binary body. diff --git a/src/hb_http_router.erl b/src/hb_http_router.erl index e6917649..ccafb322 100644 --- a/src/hb_http_router.erl +++ b/src/hb_http_router.erl @@ -3,81 +3,81 @@ -include("include/hb.hrl"). start() -> - hb_http:start(), - Dispatcher = - cowboy_router:compile( - [ - % {HostMatch, list({PathMatch, Handler, InitialState})} - {'_', [ - { - "/metrics/[:registry]", - prometheus_cowboy2_handler, - #{} - }, - { - '_', - ?MODULE, - % The default state/opts for executions from the - % HTTP API. This would be the appropriate place - % to add components (a differently scoped store, - % a different wallet, etc.) to execution for - % remote clients. - #{ - store => hb_opts:get(store), - wallet => hb_opts:get(wallet), - error_strategy => hb_opts:get(client_error_strategy) - } - } - ]} - ] - ), + hb_http:start(), + Dispatcher = + cowboy_router:compile( + [ + % {HostMatch, list({PathMatch, Handler, InitialState})} + {'_', [ + { + "/metrics/[:registry]", + prometheus_cowboy2_handler, + #{} + }, + { + '_', + ?MODULE, + % The default state/opts for executions from the + % HTTP API. This would be the appropriate place + % to add components (a differently scoped store, + % a different wallet, etc.) to execution for + % remote clients. + #{ + store => hb_opts:get(store), + wallet => hb_opts:get(wallet), + error_strategy => hb_opts:get(client_error_strategy) + } + } + ]} + ] + ), - cowboy:start_clear( - ?MODULE, - [{port, hb_opts:get(http_port)}], + cowboy:start_clear( + ?MODULE, + [{port, hb_opts:get(http_port)}], - #{ - env => #{dispatch => Dispatcher}, - metrics_callback => fun prometheus_cowboy2_instrumenter:observe/1, - stream_handlers => [cowboy_metrics_h, cowboy_stream_h] - } - ). + #{ + env => #{dispatch => Dispatcher}, + metrics_callback => fun prometheus_cowboy2_instrumenter:observe/1, + stream_handlers => [cowboy_metrics_h, cowboy_stream_h] + } + ). init(Req, State) -> - Path = cowboy_req:path(Req), - ?event({http_called_with_path, Path}), - % Note: We pass the state twice in the call below: Once for the device - % to optionally have it if useful, and once for the hb_pam module - % itself. This lets us send execution environment parameters as well as - % the request itself to the device. - case hb_pam:resolve(dev_meta, execute, [hb_http:req_to_tx(Req), State], State) of - {ok, ResultingMessage} when is_record(ResultingMessage, tx) or is_map(ResultingMessage) -> - % If the device returns a message (either normalized or not), - % we normalize and serialize it, returning it to the client. - % If the message contains a `Status` tag, we use it as the HTTP - % status code, otherwise we default to 200. - NormMessage = ar_bundles:normalize(ResultingMessage), - Signed = - case ar_bundles:is_signed(NormMessage) of - true -> NormMessage; - false -> ar_bundles:sign_item(NormMessage, hb:wallet()) - end, - hb_http:reply( - Req, - hb_http:tx_to_status(Signed), - Signed - ); - no_match -> - % If the device returns no_match, we return a 404. - ?event({could_not_match_path_to_device, Path}), - hb_http:reply( - Req, - #tx{ - tags = [{<<"Status">>, <<"500">>}], - data = <<"No matching device found.">> - } - ) - end. + Path = cowboy_req:path(Req), + ?event({http_called_with_path, Path}), + % Note: We pass the state twice in the call below: Once for the device + % to optionally have it if useful, and once for the hb_pam module + % itself. This lets us send execution environment parameters as well as + % the request itself to the device. + case hb_pam:resolve(dev_meta, execute, [hb_http:req_to_tx(Req), State], State) of + {ok, ResultingMessage} when is_record(ResultingMessage, tx) or is_map(ResultingMessage) -> + % If the device returns a message (either normalized or not), + % we normalize and serialize it, returning it to the client. + % If the message contains a `Status` tag, we use it as the HTTP + % status code, otherwise we default to 200. + NormMessage = ar_bundles:normalize(ResultingMessage), + Signed = + case ar_bundles:is_signed(NormMessage) of + true -> NormMessage; + false -> ar_bundles:sign_item(NormMessage, hb:wallet()) + end, + hb_http:reply( + Req, + hb_http:tx_to_status(Signed), + Signed + ); + no_match -> + % If the device returns no_match, we return a 404. + ?event({could_not_match_path_to_device, Path}), + hb_http:reply( + Req, + #tx{ + tags = [{<<"Status">>, <<"500">>}], + data = <<"No matching device found.">> + } + ) + end. allowed_methods(Req, State) -> - {[<<"GET">>, <<"POST">>, <<"DELETE">>], Req, State}. + {[<<"GET">>, <<"POST">>, <<"DELETE">>], Req, State}. \ No newline at end of file diff --git a/src/hb_http_signature.erl b/src/hb_http_signature.erl index ca266438..52e7c2ba 100644 --- a/src/hb_http_signature.erl +++ b/src/hb_http_signature.erl @@ -14,48 +14,48 @@ %%% authority(ComponentIdentifiers, SigParams, Key) -> - #{ - component_identifiers => ComponentIdentifiers, - sig_params => SigParams, - key => Key - }. + #{ + component_identifiers => ComponentIdentifiers, + sig_params => SigParams, + key => Key + }. sign(Authority, Req) -> - sign(Authority, Req, #{}). + sign(Authority, Req, #{}). sign(Authority, Req, Res) -> - ComponentIdentifiers = maps:get(component_identifiers, Authority), - SignatureComponentsLine = signature_components_line(ComponentIdentifiers, Req, Res), - SignatureParamsLine = signature_params_line(ComponentIdentifiers, maps:get(sig_params, Authority)), - SignatureBase = - <>, <<"\"@signature-params\": ">>, SignatureParamsLine/binary>>, - Name = random_an_binary(5), - SignatureInput = SignatureParamsLine, - % Create signature using SignatureBase and authority#key - Signature = create_signature(Authority, SignatureBase), - {ok, {Name, SignatureInput, Signature}}. + ComponentIdentifiers = maps:get(component_identifiers, Authority), + SignatureComponentsLine = signature_components_line(ComponentIdentifiers, Req, Res), + SignatureParamsLine = signature_params_line(ComponentIdentifiers, maps:get(sig_params, Authority)), + SignatureBase = + <>, <<"\"@signature-params\": ">>, SignatureParamsLine/binary>>, + Name = random_an_binary(5), + SignatureInput = SignatureParamsLine, + % Create signature using SignatureBase and authority#key + Signature = create_signature(Authority, SignatureBase), + {ok, {Name, SignatureInput, Signature}}. create_signature(Authority, SignatureBase) -> - Key = maps:get(key, Authority), - % TODO: implement - Signature = <<"SIGNED", SignatureBase/binary>>, - Signature. + Key = maps:get(key, Authority), + % TODO: implement + Signature = <<"SIGNED", SignatureBase/binary>>, + Signature. %%% %%% TODO: Ensure error cases are covered %%% - dup CI: https://datatracker.ietf.org/doc/html/rfc9421#section-2.5-7.2.2.5.2.1 %%% signature_components_line(ComponentIdentifiers, Req, Res) -> - ComponentsLine = lists:foldl( - fun(Line, Identifier) -> - % TODO: handle errors? - {ok, {I, V}} = identifier_to_component(Identifier, Req, Res), - % https://datatracker.ietf.org/doc/html/rfc9421#section-2.5-7.2.1 - <>, V/binary, <<"\n">>>> - end, - <<>>, - ComponentIdentifiers - ), - bin(ComponentsLine). + ComponentsLine = lists:foldl( + fun(Line, Identifier) -> + % TODO: handle errors? + {ok, {I, V}} = identifier_to_component(Identifier, Req, Res), + % https://datatracker.ietf.org/doc/html/rfc9421#section-2.5-7.2.1 + <>, V/binary, <<"\n">>>> + end, + <<>>, + ComponentIdentifiers + ), + bin(ComponentsLine). %%% %%% @doc construct the signature-params line part of the signature base. @@ -69,233 +69,233 @@ signature_components_line(ComponentIdentifiers, Req, Res) -> %%% See https://datatracker.ietf.org/doc/html/rfc9421#section-2.5-7.3.2.4 %%% signature_params_line(ComponentIdentifiers, SigParams) when is_map(SigParams) -> - signature_params_line(ComponentIdentifiers, maps:to_list(SigParams)); + signature_params_line(ComponentIdentifiers, maps:to_list(SigParams)); signature_params_line(ComponentIdentifiers, SigParams) when is_list(SigParams) -> - SfList = [ - { - list, - lists:map(fun sf_item/1, ComponentIdentifiers), - lists:map( - fun - ({K, V}) when is_integer(V) -> {bin(K), V}; - ({K, V}) -> {bin(K), {string, bin(V)}} - end, - SigParams - ) - } - ], - Res = hb_http_structured_fields:list(SfList), - bin(Res). + SfList = [ + { + list, + lists:map(fun sf_item/1, ComponentIdentifiers), + lists:map( + fun + ({K, V}) when is_integer(V) -> {bin(K), V}; + ({K, V}) -> {bin(K), {string, bin(V)}} + end, + SigParams + ) + } + ], + Res = hb_http_structured_fields:list(SfList), + bin(Res). identifier_to_component(Identifier = <<"@", _R/bits>>, Req, Res) -> derive_component(Identifier, Req, Res); identifier_to_component(Identifier, Req, Res) -> extract_field(Identifier, Req, Res). extract_field(Identifier, Req, Res = #{}) -> - extract_field(Identifier, Req, Res, req); + extract_field(Identifier, Req, Res, req); extract_field(Identifier, Req, Res) -> - extract_field(Identifier, Req, Res, res). + extract_field(Identifier, Req, Res, res). extract_field(Identifier, Req, Res, _Subject) -> - % The Identifier may have params and so we need to parse it - % See https://datatracker.ietf.org/doc/html/rfc9421#section-2.2-6 - {IParsed, IParams} = sf_item(Identifier), - [IsStrictFormat, IsByteSequenceEncoded, DictKey] = [ - find_sf_strict_format_param(IParams), - find_sf_byte_sequence_param(IParams), - find_sf_key_param(IParams) - ], - case (IsStrictFormat orelse DictKey) andalso IsByteSequenceEncoded of - true -> - % https://datatracker.ietf.org/doc/html/rfc9421#section-2.5-7.2.2.5.2.2 - {conflicting_params_error, <<"Component Identifier parameter 'bs' MUST not be used with 'sf' or 'key'">>}; - _ -> - Lowered = lower_bin(IParsed), - NormalizedItem = hb_http_structured_fields:item({item, {string, Lowered}, IParams}), - [IsRequestIdentifier, IsTrailerField] = [find_sf_request_param(IParams), find_sf_trailer_param(IParams)], - % There may be multiple fields that match the identifier on the Msg, - % so we filter, instead of find - MaybeRawFields = lists:filter( - fun({Key, _Value}) -> Key =:= Lowered end, - % Fields are case-insensitive, so we perform a case-insensitive search across the Msg fields - [ - {lower_bin(Key), Value} - || {Key, Value} <- maps:to_list( - maps:get( - % The field will almost certainly be a header, but could also be a trailer - % https://datatracker.ietf.org/doc/html/rfc9421#section-2.1-18.10.1 - IsTrailerField andalso trailers orelse headers, - % The header may exist on any message in the context of the signature - % which could be the Request or Response Message - % https://datatracker.ietf.org/doc/html/rfc9421#section-2.1-18.8.1 - IsRequestIdentifier andalso Req orelse Res, - #{} - ) - ) - ] - ), - case MaybeRawFields of - [] -> - % https://datatracker.ietf.org/doc/html/rfc9421#section-2.5-7.2.2.5.2.6 - {field_not_found_error, <<"Component Identifier for a field MUST be present on the message">>}; - FieldPairs -> - % The Field was found, but we still need to potentially parse it - % (it could be a Structured Field) and potentially extract - % subsequent values ie. specific dictionary key and its parameters, or further encode it - Extracted = extract_field_value( - [bin(Value) || {_Key, Value} <- FieldPairs], - [DictKey, IsStrictFormat, IsByteSequenceEncoded] - ), - {ok, {NormalizedItem, bin(Extracted)}} - end, - ok - end. + % The Identifier may have params and so we need to parse it + % See https://datatracker.ietf.org/doc/html/rfc9421#section-2.2-6 + {IParsed, IParams} = sf_item(Identifier), + [IsStrictFormat, IsByteSequenceEncoded, DictKey] = [ + find_sf_strict_format_param(IParams), + find_sf_byte_sequence_param(IParams), + find_sf_key_param(IParams) + ], + case (IsStrictFormat orelse DictKey) andalso IsByteSequenceEncoded of + true -> + % https://datatracker.ietf.org/doc/html/rfc9421#section-2.5-7.2.2.5.2.2 + {conflicting_params_error, <<"Component Identifier parameter 'bs' MUST not be used with 'sf' or 'key'">>}; + _ -> + Lowered = lower_bin(IParsed), + NormalizedItem = hb_http_structured_fields:item({item, {string, Lowered}, IParams}), + [IsRequestIdentifier, IsTrailerField] = [find_sf_request_param(IParams), find_sf_trailer_param(IParams)], + % There may be multiple fields that match the identifier on the Msg, + % so we filter, instead of find + MaybeRawFields = lists:filter( + fun({Key, _Value}) -> Key =:= Lowered end, + % Fields are case-insensitive, so we perform a case-insensitive search across the Msg fields + [ + {lower_bin(Key), Value} + || {Key, Value} <- maps:to_list( + maps:get( + % The field will almost certainly be a header, but could also be a trailer + % https://datatracker.ietf.org/doc/html/rfc9421#section-2.1-18.10.1 + IsTrailerField andalso trailers orelse headers, + % The header may exist on any message in the context of the signature + % which could be the Request or Response Message + % https://datatracker.ietf.org/doc/html/rfc9421#section-2.1-18.8.1 + IsRequestIdentifier andalso Req orelse Res, + #{} + ) + ) + ] + ), + case MaybeRawFields of + [] -> + % https://datatracker.ietf.org/doc/html/rfc9421#section-2.5-7.2.2.5.2.6 + {field_not_found_error, <<"Component Identifier for a field MUST be present on the message">>}; + FieldPairs -> + % The Field was found, but we still need to potentially parse it + % (it could be a Structured Field) and potentially extract + % subsequent values ie. specific dictionary key and its parameters, or further encode it + Extracted = extract_field_value( + [bin(Value) || {_Key, Value} <- FieldPairs], + [DictKey, IsStrictFormat, IsByteSequenceEncoded] + ), + {ok, {NormalizedItem, bin(Extracted)}} + end, + ok + end. extract_field_value(RawFields, [Key, IsStrictFormat, IsByteSequenceEncoded]) -> - % TODO: (maybe this already works?) empty string for empty header - case not (Key orelse IsStrictFormat orelse IsByteSequenceEncoded) of - % https://datatracker.ietf.org/doc/html/rfc9421#section-2.1-5 - true -> - Normalized = [trim_and_normalize(Field) || Field <- RawFields], - bin(lists:join(<<", ">>, Normalized)); - _ -> - case IsByteSequenceEncoded of - % https://datatracker.ietf.org/doc/html/rfc9421#section-2.1.3-2 - true -> - SfList = [ - {item, {binary, trim_and_normalize(Field)}, {}} - || Field <- RawFields - ], - hb_http_structured_fields:list(SfList); - _ -> - Combined = bin(lists:join(<<", ">>, RawFields)), - case sf_parse(Combined) of - % https://datatracker.ietf.org/doc/html/rfc9421#section-2.1.1-3 - {error, _} -> - {sf_parsing_error, <<"Component Identifier value could not be parsed as a structured field">>}; - {ok, SF} -> - case Key of - % Not accessing a key, so just re-serialize, which should - % properly format the data in Strict-Formatting style - false -> sf_serialize(SF); - _ -> extract_dictionary_field_value(SF, Key) - end - end - end - end. + % TODO: (maybe this already works?) empty string for empty header + case not (Key orelse IsStrictFormat orelse IsByteSequenceEncoded) of + % https://datatracker.ietf.org/doc/html/rfc9421#section-2.1-5 + true -> + Normalized = [trim_and_normalize(Field) || Field <- RawFields], + bin(lists:join(<<", ">>, Normalized)); + _ -> + case IsByteSequenceEncoded of + % https://datatracker.ietf.org/doc/html/rfc9421#section-2.1.3-2 + true -> + SfList = [ + {item, {binary, trim_and_normalize(Field)}, {}} + || Field <- RawFields + ], + hb_http_structured_fields:list(SfList); + _ -> + Combined = bin(lists:join(<<", ">>, RawFields)), + case sf_parse(Combined) of + % https://datatracker.ietf.org/doc/html/rfc9421#section-2.1.1-3 + {error, _} -> + {sf_parsing_error, <<"Component Identifier value could not be parsed as a structured field">>}; + {ok, SF} -> + case Key of + % Not accessing a key, so just re-serialize, which should + % properly format the data in Strict-Formatting style + false -> sf_serialize(SF); + _ -> extract_dictionary_field_value(SF, Key) + end + end + end + end. extract_dictionary_field_value(StructuredField = [Elem | _Rest], Key) -> - case Elem of - {Name, _} when is_binary(Name) -> - case lists:keyfind(Key, 1, StructuredField) of - % https://datatracker.ietf.org/doc/html/rfc9421#section-2.1.2-5 - false -> - {sf_key_not_found_error, <<"Component Identifier references key not found in dictionary structured field">>}; - {_, Value} -> - sf_serialize(Value) - end, - ok; - _ -> - {sf_not_dictionary_error, <<"Component Identifier cannot reference key on a non-dictionary structured field">>} - end. + case Elem of + {Name, _} when is_binary(Name) -> + case lists:keyfind(Key, 1, StructuredField) of + % https://datatracker.ietf.org/doc/html/rfc9421#section-2.1.2-5 + false -> + {sf_key_not_found_error, <<"Component Identifier references key not found in dictionary structured field">>}; + {_, Value} -> + sf_serialize(Value) + end, + ok; + _ -> + {sf_not_dictionary_error, <<"Component Identifier cannot reference key on a non-dictionary structured field">>} + end. derive_component(Identifier, Req, Res = #{}) -> - derive_component(Identifier, Req, Res, req); + derive_component(Identifier, Req, Res, req); derive_component(Identifier, Req, Res) -> - derive_component(Identifier, Req, Res, res). + derive_component(Identifier, Req, Res, res). derive_component(Identifier, Req, Res, Subject) when is_list(Identifier) -> - derive_component(list_to_binary(Identifier), Req, Res, Subject); + derive_component(list_to_binary(Identifier), Req, Res, Subject); derive_component(Identifier, Req, Res, Subject) when is_atom(Identifier) -> - derive_component(atom_to_binary(Identifier), Req, Res, Subject); + derive_component(atom_to_binary(Identifier), Req, Res, Subject); derive_component(Identifier, Req, Res, Subject) when is_binary(Identifier) -> - % The Identifier may have params and so we need to parse it - % See https://datatracker.ietf.org/doc/html/rfc9421#section-2.2-6 - {IParsed, IParams} = sf_item(Identifier), - case find_sf_request_param(IParams) andalso Subject =:= req of - % https://datatracker.ietf.org/doc/html/rfc9421#section-2.5-7.2.2.5.2.3 - true -> - {req_identifier_error, - <<"A Component Identifier may not contain a req parameter if the target is a response message">>}; - _ -> - Lowered = lower_bin(IParsed), - NormalizedItem = hb_http_structured_fields:item({item, {string, Lowered}, IParams}), - Derived = - case Lowered of - % https://datatracker.ietf.org/doc/html/rfc9421#section-2.2-4.2.1 - <<"@method">> -> - {ok, {NormalizedItem, upper_bin(maps:get(method, Req))}}; - % https://datatracker.ietf.org/doc/html/rfc9421#section-2.2-4.4.1 - <<"@target-uri">> -> - {ok, {NormalizedItem, bin(maps:get(url, Req))}}; - % https://datatracker.ietf.org/doc/html/rfc9421#section-2.2-4.6.1 - <<"@authority">> -> - URI = uri_string:parse(maps:get(url, Req)), - Authority = maps:get(host, URI), - {ok, {NormalizedItem, lower_bin(Authority)}}; - % https://datatracker.ietf.org/doc/html/rfc9421#section-2.2-4.8.1 - <<"@scheme">> -> - URI = uri_string:parse(maps:get(url, Req)), - Scheme = maps:get(schema, URI), - {ok, {NormalizedItem, lower_bin(Scheme)}}; - % https://datatracker.ietf.org/doc/html/rfc9421#section-2.2-4.10.1 - <<"@request-target">> -> - URI = uri_string:parse(maps:get(url, Req)), - % If message contains the absolute form value, then - % the value must be the absolut url - % - % TODO: maybe better way to distinguish besides a flag - % on the request? - % - % See https://datatracker.ietf.org/doc/html/rfc9421#section-2.2.5-10 - RequestTarget = - case maps:get(is_absolute_form, Req) of - true -> URI; - _ -> lists:join($?, [maps:get(path, URI), maps:get(query, URI, ?EMPTY_QUERY_PARAMS)]) - end, - {ok, {NormalizedItem, bin(RequestTarget)}}; - % https://datatracker.ietf.org/doc/html/rfc9421#section-2.2-4.12.1 - <<"@path">> -> - URI = uri_string:parse(maps:get(url, Req)), - Path = maps:get(path, URI), - {ok, {NormalizedItem, bin(Path)}}; - % https://datatracker.ietf.org/doc/html/rfc9421#section-2.2-4.14.1 - <<"@query">> -> - URI = uri_string:parse(maps:get(url, Req)), - % No query params results in a "?" value - % See https://datatracker.ietf.org/doc/html/rfc9421#section-2.2.7-14 - Query = maps:get(query, URI, ?EMPTY_QUERY_PARAMS), - {ok, {NormalizedItem, bin(Query)}}; - % https://datatracker.ietf.org/doc/html/rfc9421#section-2.2-4.16.1 - <<"@query-param">> -> - URI = uri_string:parse(maps:get(url, Req)), - case find_sf_name_param(IParams) of - % The name parameter MUST be provided when specifiying a @query-param - % Derived Component. See https://datatracker.ietf.org/doc/html/rfc9421#section-2.2.8-1 - false -> - {req_identifier_error, <<"@query_param Derived Component Identifier must specify a name parameter">>}; - Name -> - QueryParams = uri_string:dissect_query(maps:get(query, URI, "")), - QueryParam = - case lists:keyfind(Name, 1, QueryParams) of - {_, QP} -> QP; - % An missing or empty query param value results in - % an empty string value in the signature base - % https://datatracker.ietf.org/doc/html/rfc9421#section-2.2.8-4 - _ -> "" - end, - {ok, {NormalizedItem, bin(QueryParam)}} - end; - % https://datatracker.ietf.org/doc/html/rfc9421#section-2.2-4.18.1 - <<"@status">> -> - case Subject =:= req of - % https://datatracker.ietf.org/doc/html/rfc9421#section-2.2.9-8 - true -> - {res_identifier_error, <<"@status Derived Component must not be used if target is a request message">>}; - _ -> - Status = maps:get(status, Res, <<"200">>), - {ok, {NormalizedItem, Status}} - end - end, - Derived - end. + % The Identifier may have params and so we need to parse it + % See https://datatracker.ietf.org/doc/html/rfc9421#section-2.2-6 + {IParsed, IParams} = sf_item(Identifier), + case find_sf_request_param(IParams) andalso Subject =:= req of + % https://datatracker.ietf.org/doc/html/rfc9421#section-2.5-7.2.2.5.2.3 + true -> + {req_identifier_error, + <<"A Component Identifier may not contain a req parameter if the target is a response message">>}; + _ -> + Lowered = lower_bin(IParsed), + NormalizedItem = hb_http_structured_fields:item({item, {string, Lowered}, IParams}), + Derived = + case Lowered of + % https://datatracker.ietf.org/doc/html/rfc9421#section-2.2-4.2.1 + <<"@method">> -> + {ok, {NormalizedItem, upper_bin(maps:get(method, Req))}}; + % https://datatracker.ietf.org/doc/html/rfc9421#section-2.2-4.4.1 + <<"@target-uri">> -> + {ok, {NormalizedItem, bin(maps:get(url, Req))}}; + % https://datatracker.ietf.org/doc/html/rfc9421#section-2.2-4.6.1 + <<"@authority">> -> + URI = uri_string:parse(maps:get(url, Req)), + Authority = maps:get(host, URI), + {ok, {NormalizedItem, lower_bin(Authority)}}; + % https://datatracker.ietf.org/doc/html/rfc9421#section-2.2-4.8.1 + <<"@scheme">> -> + URI = uri_string:parse(maps:get(url, Req)), + Scheme = maps:get(schema, URI), + {ok, {NormalizedItem, lower_bin(Scheme)}}; + % https://datatracker.ietf.org/doc/html/rfc9421#section-2.2-4.10.1 + <<"@request-target">> -> + URI = uri_string:parse(maps:get(url, Req)), + % If message contains the absolute form value, then + % the value must be the absolut url + % + % TODO: maybe better way to distinguish besides a flag + % on the request? + % + % See https://datatracker.ietf.org/doc/html/rfc9421#section-2.2.5-10 + RequestTarget = + case maps:get(is_absolute_form, Req) of + true -> URI; + _ -> lists:join($?, [maps:get(path, URI), maps:get(query, URI, ?EMPTY_QUERY_PARAMS)]) + end, + {ok, {NormalizedItem, bin(RequestTarget)}}; + % https://datatracker.ietf.org/doc/html/rfc9421#section-2.2-4.12.1 + <<"@path">> -> + URI = uri_string:parse(maps:get(url, Req)), + Path = maps:get(path, URI), + {ok, {NormalizedItem, bin(Path)}}; + % https://datatracker.ietf.org/doc/html/rfc9421#section-2.2-4.14.1 + <<"@query">> -> + URI = uri_string:parse(maps:get(url, Req)), + % No query params results in a "?" value + % See https://datatracker.ietf.org/doc/html/rfc9421#section-2.2.7-14 + Query = maps:get(query, URI, ?EMPTY_QUERY_PARAMS), + {ok, {NormalizedItem, bin(Query)}}; + % https://datatracker.ietf.org/doc/html/rfc9421#section-2.2-4.16.1 + <<"@query-param">> -> + URI = uri_string:parse(maps:get(url, Req)), + case find_sf_name_param(IParams) of + % The name parameter MUST be provided when specifiying a @query-param + % Derived Component. See https://datatracker.ietf.org/doc/html/rfc9421#section-2.2.8-1 + false -> + {req_identifier_error, <<"@query_param Derived Component Identifier must specify a name parameter">>}; + Name -> + QueryParams = uri_string:dissect_query(maps:get(query, URI, "")), + QueryParam = + case lists:keyfind(Name, 1, QueryParams) of + {_, QP} -> QP; + % An missing or empty query param value results in + % an empty string value in the signature base + % https://datatracker.ietf.org/doc/html/rfc9421#section-2.2.8-4 + _ -> "" + end, + {ok, {NormalizedItem, bin(QueryParam)}} + end; + % https://datatracker.ietf.org/doc/html/rfc9421#section-2.2-4.18.1 + <<"@status">> -> + case Subject =:= req of + % https://datatracker.ietf.org/doc/html/rfc9421#section-2.2.9-8 + true -> + {res_identifier_error, <<"@status Derived Component must not be used if target is a request message">>}; + _ -> + Status = maps:get(status, Res, <<"200">>), + {ok, {NormalizedItem, Status}} + end + end, + Derived + end. %%% %%% Strucutured Field Utilities @@ -303,42 +303,42 @@ derive_component(Identifier, Req, Res, Subject) when is_binary(Identifier) -> sf_parse(Raw) when is_list(Raw) -> sf_parse(list_to_binary(Raw)); sf_parse(Raw) when is_binary(Raw) -> - Parsers = [], - sf_parse(Parsers, Raw). + Parsers = [], + sf_parse(Parsers, Raw). sf_parse([], _Raw) -> - {error, undefined}; + {error, undefined}; sf_parse([Parser | Rest], Raw) -> - case catch Parser(Raw) of - % skip parsers that fail - {'EXIT', _} -> sf_parse(Rest, Raw); - Parsed -> {ok, Parsed} - end. + case catch Parser(Raw) of + % skip parsers that fail + {'EXIT', _} -> sf_parse(Rest, Raw); + Parsed -> {ok, Parsed} + end. sf_serialize(StructuredField = {item, _, _}) -> - hb_http_structured_fields:item(StructuredField); + hb_http_structured_fields:item(StructuredField); sf_serialize(StructuredField = [Elem | _Rest]) -> - case Elem of - {Name, _} when is_binary(Name) -> hb_http_structured_fields:dictionary(StructuredField); - _ -> hb_http_structured_fields:list(StructuredField) - end. + case Elem of + {Name, _} when is_binary(Name) -> hb_http_structured_fields:dictionary(StructuredField); + _ -> hb_http_structured_fields:list(StructuredField) + end. sf_item({item, {_Kind, Parsed}, Params}) -> - {Parsed, Params}; + {Parsed, Params}; % TODO: should we check whether the string is already quoted? sf_item(ComponentIdentifier) when is_list(ComponentIdentifier) -> - sf_item(<<$", (lower_bin(ComponentIdentifier))/binary, $">>); + sf_item(<<$", (lower_bin(ComponentIdentifier))/binary, $">>); sf_item(ComponentIdentifier) when is_binary(ComponentIdentifier) -> - sf_item(hb_http_structured_fields:parse_item(ComponentIdentifier)). + sf_item(hb_http_structured_fields:parse_item(ComponentIdentifier)). find_sf_param(Name, Params, Default) when is_list(Name) -> - find_sf_param(list_to_binary(Name), Params, Default); + find_sf_param(list_to_binary(Name), Params, Default); find_sf_param(Name, Params, Default) -> - % [{<<"name">>,{string,<<"baz">>}}] - case lists:keyfind(Name, 1, Params) of - {_, {_, Value}} -> Value; - _ -> Default - end. + % [{<<"name">>,{string,<<"baz">>}}] + case lists:keyfind(Name, 1, Params) of + {_, {_, Value}} -> Value; + _ -> Default + end. %%% %%% https://datatracker.ietf.org/doc/html/rfc9421#section-6.5.2-1 @@ -357,7 +357,7 @@ find_sf_name_param(Params) -> find_sf_param(<<"name">>, Params, false). % https://datatracker.ietf.org/doc/html/rfc9421#section-2.1-5 trim_and_normalize(Bin) -> - binary:replace(binary:trim(Bin), <<$\n>>, <<" ">>, [global]). + binary:replace(binary:trim(Bin), <<$\n>>, <<" ">>, [global]). upper_bin(Item) when is_binary(Item) -> upper_bin(binary_to_list(Item)); upper_bin(Item) when is_list(Item) -> bin(string:uppercase(Item)). @@ -367,17 +367,17 @@ lower_bin(Item) when is_list(Item) -> bin(string:lowercase(Item)). bin(Item) when is_atom(Item) -> atom_to_binary(Item, utf8); bin(Item) when is_integer(Item) -> - case Item of - % Treat integer as an ASCII code - N when N > 0 andalso N < 256 -> <>; - N -> integer_to_binary(N) - end; + case Item of + % Treat integer as an ASCII code + N when N > 0 andalso N < 256 -> <>; + N -> integer_to_binary(N) + end; bin(Item) -> - iolist_to_binary(Item). + iolist_to_binary(Item). random_an_binary(Length) -> - Characters = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789", - ListLength = length(Characters), - RandomIndexes = [rand:uniform(ListLength) || _ <- lists:seq(1, Length)], - RandomChars = [lists:nth(Index, Characters) || Index <- RandomIndexes], - list_to_binary(RandomChars). + Characters = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789", + ListLength = length(Characters), + RandomIndexes = [rand:uniform(ListLength) || _ <- lists:seq(1, Length)], + RandomChars = [lists:nth(Index, Characters) || Index <- RandomIndexes], + list_to_binary(RandomChars). \ No newline at end of file diff --git a/src/hb_http_structured_fields.erl b/src/hb_http_structured_fields.erl index 3534d565..5c95278a 100644 --- a/src/hb_http_structured_fields.erl +++ b/src/hb_http_structured_fields.erl @@ -31,140 +31,140 @@ -type sh_dictionary() :: [{binary(), sh_item() | sh_inner_list()}]. -type sh_item() :: {item, sh_bare_item(), sh_params()}. -type sh_bare_item() :: - integer() - | sh_decimal() - | boolean() - | {string | token | binary, binary()}. + integer() + | sh_decimal() + | boolean() + | {string | token | binary, binary()}. -type sh_decimal() :: {decimal, {integer(), integer()}}. -define(IS_LC_ALPHA(C), - (C =:= $a) or (C =:= $b) or (C =:= $c) or (C =:= $d) or (C =:= $e) or - (C =:= $f) or (C =:= $g) or (C =:= $h) or (C =:= $i) or (C =:= $j) or - (C =:= $k) or (C =:= $l) or (C =:= $m) or (C =:= $n) or (C =:= $o) or - (C =:= $p) or (C =:= $q) or (C =:= $r) or (C =:= $s) or (C =:= $t) or - (C =:= $u) or (C =:= $v) or (C =:= $w) or (C =:= $x) or (C =:= $y) or - (C =:= $z) + (C =:= $a) or (C =:= $b) or (C =:= $c) or (C =:= $d) or (C =:= $e) or + (C =:= $f) or (C =:= $g) or (C =:= $h) or (C =:= $i) or (C =:= $j) or + (C =:= $k) or (C =:= $l) or (C =:= $m) or (C =:= $n) or (C =:= $o) or + (C =:= $p) or (C =:= $q) or (C =:= $r) or (C =:= $s) or (C =:= $t) or + (C =:= $u) or (C =:= $v) or (C =:= $w) or (C =:= $x) or (C =:= $y) or + (C =:= $z) ). %% Parsing. -spec parse_dictionary(binary()) -> sh_dictionary(). parse_dictionary(<<>>) -> - []; + []; parse_dictionary(<>) when ?IS_LC_ALPHA(C) or (C =:= $*) -> - parse_dict_key(R, [], <>). + parse_dict_key(R, [], <>). parse_dict_key(<<$=, $(, R0/bits>>, Acc, K) -> - {Item, R} = parse_inner_list(R0, []), - parse_dict_before_sep(R, lists:keystore(K, 1, Acc, {K, Item})); + {Item, R} = parse_inner_list(R0, []), + parse_dict_before_sep(R, lists:keystore(K, 1, Acc, {K, Item})); parse_dict_key(<<$=, R0/bits>>, Acc, K) -> - {Item, R} = parse_item1(R0), - parse_dict_before_sep(R, lists:keystore(K, 1, Acc, {K, Item})); + {Item, R} = parse_item1(R0), + parse_dict_before_sep(R, lists:keystore(K, 1, Acc, {K, Item})); parse_dict_key(<>, Acc, K) when - ?IS_LC_ALPHA(C) or ?IS_DIGIT(C) or - (C =:= $_) or (C =:= $-) or (C =:= $.) or (C =:= $*) + ?IS_LC_ALPHA(C) or ?IS_DIGIT(C) or + (C =:= $_) or (C =:= $-) or (C =:= $.) or (C =:= $*) -> - parse_dict_key(R, Acc, <>); + parse_dict_key(R, Acc, <>); parse_dict_key(<<$;, R0/bits>>, Acc, K) -> - {Params, R} = parse_before_param(R0, []), - parse_dict_before_sep(R, lists:keystore(K, 1, Acc, {K, {item, true, Params}})); + {Params, R} = parse_before_param(R0, []), + parse_dict_before_sep(R, lists:keystore(K, 1, Acc, {K, {item, true, Params}})); parse_dict_key(R, Acc, K) -> - parse_dict_before_sep(R, lists:keystore(K, 1, Acc, {K, {item, true, []}})). + parse_dict_before_sep(R, lists:keystore(K, 1, Acc, {K, {item, true, []}})). parse_dict_before_sep(<<$\s, R/bits>>, Acc) -> - parse_dict_before_sep(R, Acc); + parse_dict_before_sep(R, Acc); parse_dict_before_sep(<<$\t, R/bits>>, Acc) -> - parse_dict_before_sep(R, Acc); + parse_dict_before_sep(R, Acc); parse_dict_before_sep(<>, Acc) when C =:= $, -> - parse_dict_before_member(R, Acc); + parse_dict_before_member(R, Acc); parse_dict_before_sep(<<>>, Acc) -> - Acc. + Acc. parse_dict_before_member(<<$\s, R/bits>>, Acc) -> - parse_dict_before_member(R, Acc); + parse_dict_before_member(R, Acc); parse_dict_before_member(<<$\t, R/bits>>, Acc) -> - parse_dict_before_member(R, Acc); + parse_dict_before_member(R, Acc); parse_dict_before_member(<>, Acc) when ?IS_LC_ALPHA(C) or (C =:= $*) -> - parse_dict_key(R, Acc, <>). + parse_dict_key(R, Acc, <>). -spec parse_item(binary()) -> sh_item(). parse_item(Bin) -> - {Item, <<>>} = parse_item1(Bin), - Item. + {Item, <<>>} = parse_item1(Bin), + Item. parse_item1(Bin) -> - case parse_bare_item(Bin) of - {Item, <<$;, R/bits>>} -> - {Params, Rest} = parse_before_param(R, []), - {{item, Item, Params}, Rest}; - {Item, Rest} -> - {{item, Item, []}, Rest} - end. + case parse_bare_item(Bin) of + {Item, <<$;, R/bits>>} -> + {Params, Rest} = parse_before_param(R, []), + {{item, Item, Params}, Rest}; + {Item, Rest} -> + {{item, Item, []}, Rest} + end. -spec parse_list(binary()) -> sh_list(). parse_list(<<>>) -> - []; + []; parse_list(Bin) -> - parse_list_before_member(Bin, []). + parse_list_before_member(Bin, []). parse_list_member(<<$(, R0/bits>>, Acc) -> - {Item, R} = parse_inner_list(R0, []), - parse_list_before_sep(R, [Item | Acc]); + {Item, R} = parse_inner_list(R0, []), + parse_list_before_sep(R, [Item | Acc]); parse_list_member(R0, Acc) -> - {Item, R} = parse_item1(R0), - parse_list_before_sep(R, [Item | Acc]). + {Item, R} = parse_item1(R0), + parse_list_before_sep(R, [Item | Acc]). parse_list_before_sep(<<$\s, R/bits>>, Acc) -> - parse_list_before_sep(R, Acc); + parse_list_before_sep(R, Acc); parse_list_before_sep(<<$\t, R/bits>>, Acc) -> - parse_list_before_sep(R, Acc); + parse_list_before_sep(R, Acc); parse_list_before_sep(<<$,, R/bits>>, Acc) -> - parse_list_before_member(R, Acc); + parse_list_before_member(R, Acc); parse_list_before_sep(<<>>, Acc) -> - lists:reverse(Acc). + lists:reverse(Acc). parse_list_before_member(<<$\s, R/bits>>, Acc) -> - parse_list_before_member(R, Acc); + parse_list_before_member(R, Acc); parse_list_before_member(<<$\t, R/bits>>, Acc) -> - parse_list_before_member(R, Acc); + parse_list_before_member(R, Acc); parse_list_before_member(R, Acc) -> - parse_list_member(R, Acc). + parse_list_member(R, Acc). %% Internal. parse_inner_list(<<$\s, R/bits>>, Acc) -> - parse_inner_list(R, Acc); + parse_inner_list(R, Acc); parse_inner_list(<<$), $;, R0/bits>>, Acc) -> - {Params, R} = parse_before_param(R0, []), - {{list, lists:reverse(Acc), Params}, R}; + {Params, R} = parse_before_param(R0, []), + {{list, lists:reverse(Acc), Params}, R}; parse_inner_list(<<$), R/bits>>, Acc) -> - {{list, lists:reverse(Acc), []}, R}; + {{list, lists:reverse(Acc), []}, R}; parse_inner_list(R0, Acc) -> - {Item, R = <>} = parse_item1(R0), - true = (C =:= $\s) orelse (C =:= $)), - parse_inner_list(R, [Item | Acc]). + {Item, R = <>} = parse_item1(R0), + true = (C =:= $\s) orelse (C =:= $)), + parse_inner_list(R, [Item | Acc]). parse_before_param(<<$\s, R/bits>>, Acc) -> - parse_before_param(R, Acc); + parse_before_param(R, Acc); parse_before_param(<>, Acc) when ?IS_LC_ALPHA(C) or (C =:= $*) -> - parse_param(R, Acc, <>). + parse_param(R, Acc, <>). parse_param(<<$;, R/bits>>, Acc, K) -> - parse_before_param(R, lists:keystore(K, 1, Acc, {K, true})); + parse_before_param(R, lists:keystore(K, 1, Acc, {K, true})); parse_param(<<$=, R0/bits>>, Acc, K) -> - case parse_bare_item(R0) of - {Item, <<$;, R/bits>>} -> - parse_before_param(R, lists:keystore(K, 1, Acc, {K, Item})); - {Item, R} -> - {lists:keystore(K, 1, Acc, {K, Item}), R} - end; + case parse_bare_item(R0) of + {Item, <<$;, R/bits>>} -> + parse_before_param(R, lists:keystore(K, 1, Acc, {K, Item})); + {Item, R} -> + {lists:keystore(K, 1, Acc, {K, Item}), R} + end; parse_param(<>, Acc, K) when - ?IS_LC_ALPHA(C) or ?IS_DIGIT(C) or - (C =:= $_) or (C =:= $-) or (C =:= $.) or (C =:= $*) + ?IS_LC_ALPHA(C) or ?IS_DIGIT(C) or + (C =:= $_) or (C =:= $-) or (C =:= $.) or (C =:= $*) -> - parse_param(R, Acc, <>); + parse_param(R, Acc, <>); parse_param(R, Acc, K) -> - {lists:keystore(K, 1, Acc, {K, true}), R}. + {lists:keystore(K, 1, Acc, {K, true}), R}. %% Integer or decimal. parse_bare_item(<<$-, R/bits>>) -> parse_number(R, 0, <<$->>); @@ -180,120 +180,120 @@ parse_bare_item(<<"?0", R/bits>>) -> {false, R}; parse_bare_item(<<"?1", R/bits>>) -> {true, R}. parse_number(<>, L, Acc) when ?IS_DIGIT(C) -> - parse_number(R, L + 1, <>); + parse_number(R, L + 1, <>); parse_number(<<$., R/bits>>, L, Acc) -> - parse_decimal(R, L, 0, Acc, <<>>); + parse_decimal(R, L, 0, Acc, <<>>); parse_number(R, L, Acc) when L =< 15 -> - {binary_to_integer(Acc), R}. + {binary_to_integer(Acc), R}. parse_decimal(<>, L1, L2, IntAcc, FracAcc) when ?IS_DIGIT(C) -> - parse_decimal(R, L1, L2 + 1, IntAcc, <>); + parse_decimal(R, L1, L2 + 1, IntAcc, <>); parse_decimal(R, L1, L2, IntAcc, FracAcc0) when L1 =< 12, L2 >= 1, L2 =< 3 -> - %% While not strictly required this gives a more consistent representation. - FracAcc = - case FracAcc0 of - <<$0>> -> <<>>; - <<$0, $0>> -> <<>>; - <<$0, $0, $0>> -> <<>>; - <> -> <>; - <> -> <>; - <> -> <>; - _ -> FracAcc0 - end, - Mul = - case byte_size(FracAcc) of - 3 -> 1000; - 2 -> 100; - 1 -> 10; - 0 -> 1 - end, - Int = binary_to_integer(IntAcc), - Frac = - case FracAcc of - <<>> -> 0; - %% Mind the sign. - _ when Int < 0 -> -binary_to_integer(FracAcc); - _ -> binary_to_integer(FracAcc) - end, - {{decimal, {Int * Mul + Frac, -byte_size(FracAcc)}}, R}. + %% While not strictly required this gives a more consistent representation. + FracAcc = + case FracAcc0 of + <<$0>> -> <<>>; + <<$0, $0>> -> <<>>; + <<$0, $0, $0>> -> <<>>; + <> -> <>; + <> -> <>; + <> -> <>; + _ -> FracAcc0 + end, + Mul = + case byte_size(FracAcc) of + 3 -> 1000; + 2 -> 100; + 1 -> 10; + 0 -> 1 + end, + Int = binary_to_integer(IntAcc), + Frac = + case FracAcc of + <<>> -> 0; + %% Mind the sign. + _ when Int < 0 -> -binary_to_integer(FracAcc); + _ -> binary_to_integer(FracAcc) + end, + {{decimal, {Int * Mul + Frac, -byte_size(FracAcc)}}, R}. parse_string(<<$\\, $", R/bits>>, Acc) -> - parse_string(R, <>); + parse_string(R, <>); parse_string(<<$\\, $\\, R/bits>>, Acc) -> - parse_string(R, <>); + parse_string(R, <>); parse_string(<<$", R/bits>>, Acc) -> - {{string, Acc}, R}; + {{string, Acc}, R}; parse_string(<>, Acc) when - C >= 16#20, C =< 16#21; - C >= 16#23, C =< 16#5b; - C >= 16#5d, C =< 16#7e + C >= 16#20, C =< 16#21; + C >= 16#23, C =< 16#5b; + C >= 16#5d, C =< 16#7e -> - parse_string(R, <>). + parse_string(R, <>). parse_token(<>, Acc) when ?IS_TOKEN(C) or (C =:= $:) or (C =:= $/) -> - parse_token(R, <>); + parse_token(R, <>); parse_token(R, Acc) -> - {{token, Acc}, R}. + {{token, Acc}, R}. parse_binary(<<$:, R/bits>>, Acc) -> - {{binary, base64:decode(Acc)}, R}; + {{binary, base64:decode(Acc)}, R}; parse_binary(<>, Acc) when ?IS_ALPHANUM(C) or (C =:= $+) or (C =:= $/) or (C =:= $=) -> - parse_binary(R, <>). + parse_binary(R, <>). -ifdef(TEST). parse_struct_hd_test_() -> - Files = filelib:wildcard("deps/structured-header-tests/*.json"), - lists:flatten([ - begin - {ok, JSON} = file:read_file(File), - Tests = jsx:decode(JSON, [return_maps]), - [ - {iolist_to_binary(io_lib:format("~s: ~s", [filename:basename(File), Name])), fun() -> - %% The implementation is strict. We fail whenever we can. - CanFail = maps:get(<<"can_fail">>, Test, false), - MustFail = maps:get(<<"must_fail">>, Test, false), - io:format( - "must fail ~p~nexpected json ~0p~n", - [MustFail, maps:get(<<"expected">>, Test, undefined)] - ), - Expected = - case MustFail of - true -> undefined; - false -> expected_to_term(maps:get(<<"expected">>, Test)) - end, - io:format("expected term: ~0p", [Expected]), - Raw = raw_to_binary(Raw0), - case HeaderType of - <<"dictionary">> when MustFail; CanFail -> - {'EXIT', _} = (catch parse_dictionary(Raw)); - %% The test "binary.json: non-zero pad bits" does not fail - %% due to our reliance on Erlang/OTP's base64 module. - <<"item">> when CanFail -> - case (catch parse_item(Raw)) of - {'EXIT', _} -> ok; - Expected -> ok - end; - <<"item">> when MustFail -> - {'EXIT', _} = (catch parse_item(Raw)); - <<"list">> when MustFail; CanFail -> - {'EXIT', _} = (catch parse_list(Raw)); - <<"dictionary">> -> - Expected = (catch parse_dictionary(Raw)); - <<"item">> -> - Expected = (catch parse_item(Raw)); - <<"list">> -> - Expected = (catch parse_list(Raw)) - end - end} - || Test = #{ - <<"name">> := Name, - <<"header_type">> := HeaderType, - <<"raw">> := Raw0 - } <- Tests - ] - end - || File <- Files - ]). + Files = filelib:wildcard("deps/structured-header-tests/*.json"), + lists:flatten([ + begin + {ok, JSON} = file:read_file(File), + Tests = jsx:decode(JSON, [return_maps]), + [ + {iolist_to_binary(io_lib:format("~s: ~s", [filename:basename(File), Name])), fun() -> + %% The implementation is strict. We fail whenever we can. + CanFail = maps:get(<<"can_fail">>, Test, false), + MustFail = maps:get(<<"must_fail">>, Test, false), + io:format( + "must fail ~p~nexpected json ~0p~n", + [MustFail, maps:get(<<"expected">>, Test, undefined)] + ), + Expected = + case MustFail of + true -> undefined; + false -> expected_to_term(maps:get(<<"expected">>, Test)) + end, + io:format("expected term: ~0p", [Expected]), + Raw = raw_to_binary(Raw0), + case HeaderType of + <<"dictionary">> when MustFail; CanFail -> + {'EXIT', _} = (catch parse_dictionary(Raw)); + %% The test "binary.json: non-zero pad bits" does not fail + %% due to our reliance on Erlang/OTP's base64 module. + <<"item">> when CanFail -> + case (catch parse_item(Raw)) of + {'EXIT', _} -> ok; + Expected -> ok + end; + <<"item">> when MustFail -> + {'EXIT', _} = (catch parse_item(Raw)); + <<"list">> when MustFail; CanFail -> + {'EXIT', _} = (catch parse_list(Raw)); + <<"dictionary">> -> + Expected = (catch parse_dictionary(Raw)); + <<"item">> -> + Expected = (catch parse_item(Raw)); + <<"list">> -> + Expected = (catch parse_list(Raw)) + end + end} + || Test = #{ + <<"name">> := Name, + <<"header_type">> := HeaderType, + <<"raw">> := Raw0 + } <- Tests + ] + end + || File <- Files + ]). %% The tests JSON use arrays for almost everything. Identifying %% what is what requires looking deeper in the values: @@ -306,55 +306,55 @@ parse_struct_hd_test_() -> %% Item. expected_to_term([Bare, []]) when - is_boolean(Bare); is_number(Bare); is_binary(Bare); is_map(Bare) + is_boolean(Bare); is_number(Bare); is_binary(Bare); is_map(Bare) -> - {item, e2tb(Bare), []}; + {item, e2tb(Bare), []}; expected_to_term([Bare, Params = [[<<_/bits>>, _] | _]]) when - is_boolean(Bare); is_number(Bare); is_binary(Bare); is_map(Bare) + is_boolean(Bare); is_number(Bare); is_binary(Bare); is_map(Bare) -> - {item, e2tb(Bare), e2tp(Params)}; + {item, e2tb(Bare), e2tp(Params)}; %% Empty list or dictionary. expected_to_term([]) -> - []; + []; %% Dictionary. %% %% We exclude empty list from values because that could %% be confused with an outer list of strings. There is %% currently no conflicts in the tests thankfully. expected_to_term(Dict = [[<<_/bits>>, V] | _]) when V =/= [] -> - e2t(Dict); + e2t(Dict); %% Outer list. expected_to_term(List) when is_list(List) -> - [e2t(E) || E <- List]. + [e2t(E) || E <- List]. %% Dictionary. e2t(Dict = [[<<_/bits>>, _] | _]) -> - [{K, e2t(V)} || [K, V] <- Dict]; + [{K, e2t(V)} || [K, V] <- Dict]; %% Inner list. e2t([List, Params]) when is_list(List) -> - {list, [e2t(E) || E <- List], e2tp(Params)}; + {list, [e2t(E) || E <- List], e2tp(Params)}; %% Item. e2t([Bare, Params]) -> - {item, e2tb(Bare), e2tp(Params)}. + {item, e2tb(Bare), e2tp(Params)}. %% Bare item. e2tb(#{<<"__type">> := <<"token">>, <<"value">> := V}) -> - {token, V}; + {token, V}; e2tb(#{<<"__type">> := <<"binary">>, <<"value">> := V}) -> - {binary, base32:decode(V)}; + {binary, base32:decode(V)}; e2tb(V) when is_binary(V) -> - {string, V}; + {string, V}; e2tb(V) when is_float(V) -> - %% There should be no rounding needed for the test cases. - {decimal, decimal:to_decimal(V, #{precision => 3, rounding => round_down})}; + %% There should be no rounding needed for the test cases. + {decimal, decimal:to_decimal(V, #{precision => 3, rounding => round_down})}; e2tb(V) -> - V. + V. %% Params. e2tp([]) -> - []; + []; e2tp(Params) -> - [{K, e2tb(V)} || [K, V] <- Params]. + [{K, e2tb(V)} || [K, V] <- Params]. %% The Cowlib parsers currently do not support resuming parsing %% in the case of multiple headers. To make tests work we modify @@ -364,132 +364,132 @@ e2tp(Params) -> %% Similarly, the Cowlib parsers expect the leading and trailing %% whitespace to be removed before calling the parser. raw_to_binary(RawList) -> - trim_ws(iolist_to_binary(lists:join(<<", ">>, RawList))). + trim_ws(iolist_to_binary(lists:join(<<", ">>, RawList))). trim_ws(<<$\s, R/bits>>) -> trim_ws(R); trim_ws(R) -> trim_ws_end(R, byte_size(R) - 1). trim_ws_end(_, -1) -> - <<>>; + <<>>; trim_ws_end(Value, N) -> - case binary:at(Value, N) of - $\s -> - trim_ws_end(Value, N - 1); - _ -> - S = N + 1, - <> = Value, - Value2 - end. + case binary:at(Value, N) of + $\s -> + trim_ws_end(Value, N - 1); + _ -> + S = N + 1, + <> = Value, + Value2 + end. -endif. %% Building. -spec dictionary(#{binary() => sh_item() | sh_inner_list()} | sh_dictionary()) -> - iolist(). + iolist(). dictionary(Map) when is_map(Map) -> - dictionary(maps:to_list(Map)); + dictionary(maps:to_list(Map)); dictionary(KVList) when is_list(KVList) -> - lists:join(<<", ">>, [ - case Value of - true -> Key; - _ -> [Key, $=, item_or_inner_list(Value)] - end - || {Key, Value} <- KVList - ]). + lists:join(<<", ">>, [ + case Value of + true -> Key; + _ -> [Key, $=, item_or_inner_list(Value)] + end + || {Key, Value} <- KVList + ]). -spec item(sh_item()) -> iolist(). item({item, BareItem, Params}) -> - [bare_item(BareItem), params(Params)]. + [bare_item(BareItem), params(Params)]. -spec list(sh_list()) -> iolist(). list(List) -> - lists:join(<<", ">>, [item_or_inner_list(Value) || Value <- List]). + lists:join(<<", ">>, [item_or_inner_list(Value) || Value <- List]). item_or_inner_list(Value = {list, _, _}) -> - inner_list(Value); + inner_list(Value); item_or_inner_list(Value) -> - item(Value). + item(Value). inner_list({list, List, Params}) -> - [$(, lists:join($\s, [item(Value) || Value <- List]), $), params(Params)]. + [$(, lists:join($\s, [item(Value) || Value <- List]), $), params(Params)]. bare_item({string, String}) -> - [$", escape_string(String, <<>>), $"]; + [$", escape_string(String, <<>>), $"]; %% @todo Must fail if Token has invalid characters. bare_item({token, Token}) -> - Token; + Token; bare_item({binary, Binary}) -> - [$:, base64:encode(Binary), $:]; + [$:, base64:encode(Binary), $:]; bare_item({decimal, {Base, Exp}}) when Exp >= 0 -> - Mul = - case Exp of - 0 -> 1; - 1 -> 10; - 2 -> 100; - 3 -> 1000; - 4 -> 10000; - 5 -> 100000; - 6 -> 1000000; - 7 -> 10000000; - 8 -> 100000000; - 9 -> 1000000000; - 10 -> 10000000000; - 11 -> 100000000000; - 12 -> 1000000000000 - end, - MaxLenWithSign = - if - Base < 0 -> 13; - true -> 12 - end, - Bin = integer_to_binary(Base * Mul), - true = byte_size(Bin) =< MaxLenWithSign, - [Bin, <<".0">>]; + Mul = + case Exp of + 0 -> 1; + 1 -> 10; + 2 -> 100; + 3 -> 1000; + 4 -> 10000; + 5 -> 100000; + 6 -> 1000000; + 7 -> 10000000; + 8 -> 100000000; + 9 -> 1000000000; + 10 -> 10000000000; + 11 -> 100000000000; + 12 -> 1000000000000 + end, + MaxLenWithSign = + if + Base < 0 -> 13; + true -> 12 + end, + Bin = integer_to_binary(Base * Mul), + true = byte_size(Bin) =< MaxLenWithSign, + [Bin, <<".0">>]; bare_item({decimal, {Base, -1}}) -> - Int = Base div 10, - Frac = abs(Base) rem 10, - [integer_to_binary(Int), $., integer_to_binary(Frac)]; + Int = Base div 10, + Frac = abs(Base) rem 10, + [integer_to_binary(Int), $., integer_to_binary(Frac)]; bare_item({decimal, {Base, -2}}) -> - Int = Base div 100, - Frac = abs(Base) rem 100, - [integer_to_binary(Int), $., integer_to_binary(Frac)]; + Int = Base div 100, + Frac = abs(Base) rem 100, + [integer_to_binary(Int), $., integer_to_binary(Frac)]; bare_item({decimal, {Base, -3}}) -> - Int = Base div 1000, - Frac = abs(Base) rem 1000, - [integer_to_binary(Int), $., integer_to_binary(Frac)]; + Int = Base div 1000, + Frac = abs(Base) rem 1000, + [integer_to_binary(Int), $., integer_to_binary(Frac)]; bare_item({decimal, {Base, Exp}}) -> - Div = exp_div(Exp), - Int0 = Base div Div, - true = abs(Int0) < 1000000000000, - Frac0 = abs(Base) rem Div, - DivFrac = Div div 1000, - Frac1 = Frac0 div DivFrac, - {Int, Frac} = - if - (Frac0 rem DivFrac) > (DivFrac div 2) -> - case Frac1 of - 999 when Int0 < 0 -> {Int0 - 1, 0}; - 999 -> {Int0 + 1, 0}; - _ -> {Int0, Frac1 + 1} - end; - true -> - {Int0, Frac1} - end, - [ - integer_to_binary(Int), - $., - if - Frac < 10 -> [$0, $0, integer_to_binary(Frac)]; - Frac < 100 -> [$0, integer_to_binary(Frac)]; - true -> integer_to_binary(Frac) - end - ]; + Div = exp_div(Exp), + Int0 = Base div Div, + true = abs(Int0) < 1000000000000, + Frac0 = abs(Base) rem Div, + DivFrac = Div div 1000, + Frac1 = Frac0 div DivFrac, + {Int, Frac} = + if + (Frac0 rem DivFrac) > (DivFrac div 2) -> + case Frac1 of + 999 when Int0 < 0 -> {Int0 - 1, 0}; + 999 -> {Int0 + 1, 0}; + _ -> {Int0, Frac1 + 1} + end; + true -> + {Int0, Frac1} + end, + [ + integer_to_binary(Int), + $., + if + Frac < 10 -> [$0, $0, integer_to_binary(Frac)]; + Frac < 100 -> [$0, integer_to_binary(Frac)]; + true -> integer_to_binary(Frac) + end + ]; bare_item(Integer) when is_integer(Integer) -> - integer_to_binary(Integer); + integer_to_binary(Integer); bare_item(true) -> - <<"?1">>; + <<"?1">>; bare_item(false) -> - <<"?0">>. + <<"?0">>. exp_div(0) -> 1; exp_div(N) -> 10 * exp_div(N + 1). @@ -500,43 +500,43 @@ escape_string(<<$", R/bits>>, Acc) -> escape_string(R, <>); escape_string(<>, Acc) -> escape_string(R, <>). params(Params) -> - [ - case Param of - {Key, true} -> [$;, Key]; - {Key, Value} -> [$;, Key, $=, bare_item(Value)] - end - || Param <- Params - ]. + [ + case Param of + {Key, true} -> [$;, Key]; + {Key, Value} -> [$;, Key, $=, bare_item(Value)] + end + || Param <- Params + ]. -ifdef(TEST). struct_hd_identity_test_() -> - Files = filelib:wildcard("deps/structured-header-tests/*.json"), - lists:flatten([ - begin - {ok, JSON} = file:read_file(File), - Tests = jsx:decode(JSON, [return_maps]), - [ - {iolist_to_binary(io_lib:format("~s: ~s", [filename:basename(File), Name])), fun() -> - io:format("expected json ~0p~n", [Expected0]), - Expected = expected_to_term(Expected0), - io:format("expected term: ~0p", [Expected]), - case HeaderType of - <<"dictionary">> -> - Expected = parse_dictionary(iolist_to_binary(dictionary(Expected))); - <<"item">> -> - Expected = parse_item(iolist_to_binary(item(Expected))); - <<"list">> -> - Expected = parse_list(iolist_to_binary(list(Expected))) - end - end} - || #{ - <<"name">> := Name, - <<"header_type">> := HeaderType, - %% We only run tests that must not fail. - <<"expected">> := Expected0 - } <- Tests - ] - end - || File <- Files - ]). --endif. + Files = filelib:wildcard("deps/structured-header-tests/*.json"), + lists:flatten([ + begin + {ok, JSON} = file:read_file(File), + Tests = jsx:decode(JSON, [return_maps]), + [ + {iolist_to_binary(io_lib:format("~s: ~s", [filename:basename(File), Name])), fun() -> + io:format("expected json ~0p~n", [Expected0]), + Expected = expected_to_term(Expected0), + io:format("expected term: ~0p", [Expected]), + case HeaderType of + <<"dictionary">> -> + Expected = parse_dictionary(iolist_to_binary(dictionary(Expected))); + <<"item">> -> + Expected = parse_item(iolist_to_binary(item(Expected))); + <<"list">> -> + Expected = parse_list(iolist_to_binary(list(Expected))) + end + end} + || #{ + <<"name">> := Name, + <<"header_type">> := HeaderType, + %% We only run tests that must not fail. + <<"expected">> := Expected0 + } <- Tests + ] + end + || File <- Files + ]). +-endif. \ No newline at end of file diff --git a/src/hb_message.erl b/src/hb_message.erl index ab1c05b2..907a8ec1 100644 --- a/src/hb_message.erl +++ b/src/hb_message.erl @@ -21,15 +21,15 @@ -define(MAX_TAG_VAL, 128). %% @doc The list of TX fields that users can set directly. -define(TX_KEYS, - [id, unsigned_id, last_tx, owner, target, signature]). + [id, unsigned_id, last_tx, owner, target, signature]). -define(FILTERED_TAGS, - [ - <<"PAM-Large-Binary">>, - <<"Bundle-Format">>, - <<"Bundle-Map">>, - <<"Bundle-Version">>, - <<"Type:">> - ] + [ + <<"PAM-Large-Binary">>, + <<"Bundle-Format">>, + <<"Bundle-Map">>, + <<"Bundle-Version">>, + <<"Type:">> + ] ). -define(REGEN_KEYS, [id, unsigned_id]). @@ -42,49 +42,49 @@ print(Msg, Indent) -> %% to start from. format(Item) -> format(Item, 0). format(Bin, Indent) when is_binary(Bin) -> - hb_util:format_indented( - hb_util:format_binary(Bin), - Indent - ); + hb_util:format_indented( + hb_util:format_binary(Bin), + Indent + ); format(Map, Indent) when is_map(Map) -> Header = hb_util:format_indented("Message {~n", Indent), Res = lists:map( fun({Key, Val}) -> - NormKey = hb_pam:to_key(Key, #{ error_strategy => ignore }), - KeyStr = - case NormKey of - Key -> - io_lib:format("~p", [NormKey]); - undefined -> - io_lib:format("~p [!!! INVALID KEY !!!]", [Key]); - _ -> - io_lib:format("~p [raw: ~p]", [NormKey, Key]) - end, - hb_util:format_indented( - "~s := ~s~n", - [ - lists:flatten(KeyStr), - case Val of - NextMap when is_map(NextMap) -> - hb_util:format_map(NextMap, Indent + 2); - Bin when is_binary(Bin) -> - hb_util:format_binary(Bin); - Other -> - io_lib:format("~p", [Other]) - end - ], - Indent + 1 - ) - end, + NormKey = hb_pam:to_key(Key, #{ error_strategy => ignore }), + KeyStr = + case NormKey of + Key -> + io_lib:format("~p", [NormKey]); + undefined -> + io_lib:format("~p [!!! INVALID KEY !!!]", [Key]); + _ -> + io_lib:format("~p [raw: ~p]", [NormKey, Key]) + end, + hb_util:format_indented( + "~s := ~s~n", + [ + lists:flatten(KeyStr), + case Val of + NextMap when is_map(NextMap) -> + hb_util:format_map(NextMap, Indent + 2); + Bin when is_binary(Bin) -> + hb_util:format_binary(Bin); + Other -> + io_lib:format("~p", [Other]) + end + ], + Indent + 1 + ) + end, maps:to_list(Map) ), - case Res of - [] -> "[Empty map]"; - _ -> - lists:flatten( - Header ++ Res ++ hb_util:format_indented("}", Indent) - ) - end; + case Res of + [] -> "[Empty map]"; + _ -> + lists:flatten( + Header ++ Res ++ hb_util:format_indented("}", Indent) + ) + end; format(Item, Indent) -> % Whatever we have is not a message map. hb_util:format_indented("[UNEXPECTED VALUE] ~p", [Item], Indent). @@ -92,101 +92,101 @@ format(Item, Indent) -> %% @doc Return the signers of a message. For now, this is just the signer %% of the message itself. In the future, we will support multiple signers. signers(Msg) -> - [ar_bundles:signer(Msg)]. + [ar_bundles:signer(Msg)]. load(Store, ID) when is_binary(ID) - andalso (byte_size(ID) == 43 orelse byte_size(ID) == 32) -> - hb_cache:read_message(Store, ID); + andalso (byte_size(ID) == 43 orelse byte_size(ID) == 32) -> + hb_cache:read_message(Store, ID); load(Store, Path) -> - hb_cache:read(Store, Path). + hb_cache:read(Store, Path). %% @doc Check if two maps match, including recursively checking nested maps. match(Map1, Map2) -> - Keys1 = maps:keys(NormMap1 = minimize(normalize(Map1))), - Keys2 = maps:keys(NormMap2 = minimize(normalize(Map2))), - case Keys1 == Keys2 of - true -> - lists:all( - fun(Key) -> - Val1 = maps:get(Key, NormMap1, not_found), - Val2 = maps:get(Key, NormMap2, not_found), - case is_map(Val1) andalso is_map(Val2) of - true -> match(Val1, Val2); - false -> - case Val1 == Val2 of - true -> true; - false -> - ?event( - {key_mismatch, - {explicit, {Key, Val1, Val2}} - } - ), - false - end - end - end, - Keys1 - ); - false -> - ?event({keys_mismatch, Keys1, Keys2}), - false - end. + Keys1 = maps:keys(NormMap1 = minimize(normalize(Map1))), + Keys2 = maps:keys(NormMap2 = minimize(normalize(Map2))), + case Keys1 == Keys2 of + true -> + lists:all( + fun(Key) -> + Val1 = maps:get(Key, NormMap1, not_found), + Val2 = maps:get(Key, NormMap2, not_found), + case is_map(Val1) andalso is_map(Val2) of + true -> match(Val1, Val2); + false -> + case Val1 == Val2 of + true -> true; + false -> + ?event( + {key_mismatch, + {explicit, {Key, Val1, Val2}} + } + ), + false + end + end + end, + Keys1 + ); + false -> + ?event({keys_mismatch, Keys1, Keys2}), + false + end. matchable_keys(Map) -> - lists:sort(lists:map(fun hb_pam:key_to_binary/1, maps:keys(Map))). + lists:sort(lists:map(fun hb_pam:key_to_binary/1, maps:keys(Map))). %% @doc Normalize the keys in a map. Also takes a list of keys and returns a %% sorted list of normalized keys if the input is a list. normalize_keys(Keys) when is_list(Keys) -> - lists:sort(lists:map(fun hb_pam:key_to_binary/1, Keys)); + lists:sort(lists:map(fun hb_pam:key_to_binary/1, Keys)); normalize_keys(Map) -> - maps:from_list( - lists:map( - fun({Key, Value}) -> - {hb_pam:key_to_binary(Key), Value} - end, - maps:to_list(Map) - ) - ). + maps:from_list( + lists:map( + fun({Key, Value}) -> + {hb_pam:key_to_binary(Key), Value} + end, + maps:to_list(Map) + ) + ). %% @doc Remove keys from the map that can be regenerated. minimize(Map) -> - NormRegenKeys = normalize_keys(?REGEN_KEYS), - maps:filter( - fun(Key, _) -> - not lists:member(hb_pam:key_to_binary(Key), NormRegenKeys) - end, - Map - ). + NormRegenKeys = normalize_keys(?REGEN_KEYS), + maps:filter( + fun(Key, _) -> + not lists:member(hb_pam:key_to_binary(Key), NormRegenKeys) + end, + Map + ). %% @doc Return a map with only the keys that necessary, without those that can %% be regenerated. normalize(Map) -> - NormalizedMap = normalize_keys(Map), - FilteredMap = filter_default_tx_keys(NormalizedMap), - maps:with(matchable_keys(FilteredMap), FilteredMap). + NormalizedMap = normalize_keys(Map), + FilteredMap = filter_default_tx_keys(NormalizedMap), + maps:with(matchable_keys(FilteredMap), FilteredMap). %% @doc Remove keys from a map that have the default values found in the tx %% record. filter_default_tx_keys(Map) -> - DefaultsMap = default_tx_message(), - maps:filter( - fun(Key, Value) -> - case maps:find(hb_pam:key_to_binary(Key), DefaultsMap) of - {ok, Value} -> false; - _ -> true - end - end, - Map - ). + DefaultsMap = default_tx_message(), + maps:filter( + fun(Key, Value) -> + case maps:find(hb_pam:key_to_binary(Key), DefaultsMap) of + {ok, Value} -> false; + _ -> true + end + end, + Map + ). %% @doc Get the normalized fields and default values of the tx record. default_tx_message() -> - normalize_keys(maps:from_list(default_tx_list())). + normalize_keys(maps:from_list(default_tx_list())). %% @doc Get the ordered list of fields and default values of the tx record. default_tx_list() -> - lists:zip(record_info(fields, tx), tl(tuple_to_list(#tx{}))). + lists:zip(record_info(fields, tx), tl(tuple_to_list(#tx{}))). %% @doc Serialize a message to a binary representation, either as JSON or the %% binary format native to the message/bundles spec in use. @@ -194,15 +194,15 @@ serialize(M) -> serialize(M, binary). serialize(M, json) -> jiffy:encode(ar_bundles:item_to_json_struct(M)); serialize(M, binary) -> - ar_bundles:serialize(message_to_tx(M)). + ar_bundles:serialize(message_to_tx(M)). %% @doc Deserialize a message from a binary representation. deserialize(B) -> deserialize(B, binary). deserialize(J, json) -> - {JSONStruct} = jiffy:decode(J), - ar_bundles:json_struct_to_item(JSONStruct); + {JSONStruct} = jiffy:decode(J), + ar_bundles:json_struct_to_item(JSONStruct); deserialize(B, binary) -> - tx_to_message(ar_bundles:deserialize(B)). + tx_to_message(ar_bundles:deserialize(B)). %% @doc Internal helper to translate a message to its #tx record representation, %% which can then be used by ar_bundles to serialize the message. We call the @@ -210,516 +210,516 @@ deserialize(B, binary) -> %% do this recursively to handle nested messages. The base case is that we hit %% a binary, which we return as is. message_to_tx(Binary) when is_binary(Binary) -> - % ar_bundles cannot serialize just a simple binary or get an ID for it, so - % so we turn it into a TX record with a special tag, tx_to_message will - % identify this tag and extract just the binary. - #tx{ - tags= [{<<"PAM-Large-Binary">>, integer_to_binary(byte_size(Binary))}], - data = Binary - }; + % ar_bundles cannot serialize just a simple binary or get an ID for it, so + % so we turn it into a TX record with a special tag, tx_to_message will + % identify this tag and extract just the binary. + #tx{ + tags= [{<<"PAM-Large-Binary">>, integer_to_binary(byte_size(Binary))}], + data = Binary + }; message_to_tx(TX) when is_record(TX, tx) -> TX; message_to_tx(M) when is_map(M) -> - % Get the keys that will be serialized, excluding private keys. Note that - % we do not call hb_pam:resolve here because we want to include all keys - % in the underlying map, except the private ones. - Keys = maps:keys(M), - % Translate the keys into a binary map. If a key has a value that is a map, - % we recursively turn its children into messages. Notably, we do not simply - % call message_to_tx/1 on the inner map because that would lead to adding - % an extra layer of nesting to the data. - %?event({message_to_tx, {keys, Keys}, {map, M}}), - MsgKeyMap = - maps:from_list(lists:flatten( - lists:map( - fun(Key) -> - case maps:find(Key, M) of - {ok, Map} when is_map(Map) -> - {Key, message_to_tx(Map)}; - {ok, Value} when is_binary(Value) -> - {Key, Value}; - {ok, Value} when is_atom(Value) or is_integer(Value) -> - ItemKey = hb_pam:key_to_binary(Key), - {Type, BinaryValue} = encode_value(Value), - [ - {<<"Type:", ItemKey/binary>>, Type}, - {ItemKey, BinaryValue} - ]; - {ok, _} -> - [] - end - end, - lists:filter( - fun(Key) -> - % Filter keys that the user could set directly, but - % should be regenerated when moving msg -> TX, as well - % as private keys. - not lists:member(Key, ?REGEN_KEYS) andalso - not hb_private:is_private(Key) - end, - Keys - ) - ) - )), - NormalizedMsgKeyMap = normalize_keys(MsgKeyMap), - % Iterate through the default fields, replacing them with the values from - % the message map if they are present. - {RemainingMap, BaseTXList} = lists:foldl( - fun({Field, Default}, {RemMap, Acc}) -> - NormKey = hb_pam:key_to_binary(Field), - case maps:find(NormKey, NormalizedMsgKeyMap) of - error -> {RemMap, [Default | Acc]}; - {ok, Value} -> {maps:remove(NormKey, RemMap), [Value | Acc]} - end - end, - {NormalizedMsgKeyMap, []}, - default_tx_list() - ), - % Rebuild the tx record from the new list of fields and values. - TXWithoutTags = list_to_tuple([tx | lists:reverse(BaseTXList)]), - % Calculate which set of the remaining keys will be used as tags. - {Tags, RawDataItems} = - lists:partition( - fun({_Key, Value}) when byte_size(Value) =< ?MAX_TAG_VAL -> true; - (_) -> false - end, - [ - {Key, maps:get(Key, RemainingMap)} - || - Key <- maps:keys(RemainingMap) - ] - ), - % We don't let the user set the tags directly, but they can instead set any - % number of keys to short binary values, which will be included as tags. - TX = TXWithoutTags#tx { tags = Tags }, - % Recursively turn the remaining data items into tx records. - DataItems = maps:from_list(lists:map( - fun({Key, Value}) -> - {Key, message_to_tx(Value)} - end, - RawDataItems - )), - % Set the data based on the remaining keys. - TXWithData = - case {TX#tx.data, maps:size(DataItems)} of - {Binary, 0} when is_binary(Binary) -> - TX; - {?DEFAULT_DATA, _} -> - TX#tx { data = DataItems }; - {Data, _} when is_map(Data) -> - TX#tx { data = maps:merge(Data, DataItems) }; - {Data, _} when is_record(Data, tx) -> - TX#tx { data = DataItems#{ data => Data } }; - {Data, _} when is_binary(Data) -> - TX#tx { data = DataItems#{ data => message_to_tx(Data) } } - end, - % ?event({prepared_tx_before_ids, - % {tags, {explicit, TXWithData#tx.tags}}, - % {data, TXWithData#tx.data} - % }), - ar_bundles:reset_ids(ar_bundles:normalize(TXWithData)); + % Get the keys that will be serialized, excluding private keys. Note that + % we do not call hb_pam:resolve here because we want to include all keys + % in the underlying map, except the private ones. + Keys = maps:keys(M), + % Translate the keys into a binary map. If a key has a value that is a map, + % we recursively turn its children into messages. Notably, we do not simply + % call message_to_tx/1 on the inner map because that would lead to adding + % an extra layer of nesting to the data. + %?event({message_to_tx, {keys, Keys}, {map, M}}), + MsgKeyMap = + maps:from_list(lists:flatten( + lists:map( + fun(Key) -> + case maps:find(Key, M) of + {ok, Map} when is_map(Map) -> + {Key, message_to_tx(Map)}; + {ok, Value} when is_binary(Value) -> + {Key, Value}; + {ok, Value} when is_atom(Value) or is_integer(Value) -> + ItemKey = hb_pam:key_to_binary(Key), + {Type, BinaryValue} = encode_value(Value), + [ + {<<"Type:", ItemKey/binary>>, Type}, + {ItemKey, BinaryValue} + ]; + {ok, _} -> + [] + end + end, + lists:filter( + fun(Key) -> + % Filter keys that the user could set directly, but + % should be regenerated when moving msg -> TX, as well + % as private keys. + not lists:member(Key, ?REGEN_KEYS) andalso + not hb_private:is_private(Key) + end, + Keys + ) + ) + )), + NormalizedMsgKeyMap = normalize_keys(MsgKeyMap), + % Iterate through the default fields, replacing them with the values from + % the message map if they are present. + {RemainingMap, BaseTXList} = lists:foldl( + fun({Field, Default}, {RemMap, Acc}) -> + NormKey = hb_pam:key_to_binary(Field), + case maps:find(NormKey, NormalizedMsgKeyMap) of + error -> {RemMap, [Default | Acc]}; + {ok, Value} -> {maps:remove(NormKey, RemMap), [Value | Acc]} + end + end, + {NormalizedMsgKeyMap, []}, + default_tx_list() + ), + % Rebuild the tx record from the new list of fields and values. + TXWithoutTags = list_to_tuple([tx | lists:reverse(BaseTXList)]), + % Calculate which set of the remaining keys will be used as tags. + {Tags, RawDataItems} = + lists:partition( + fun({_Key, Value}) when byte_size(Value) =< ?MAX_TAG_VAL -> true; + (_) -> false + end, + [ + {Key, maps:get(Key, RemainingMap)} + || + Key <- maps:keys(RemainingMap) + ] + ), + % We don't let the user set the tags directly, but they can instead set any + % number of keys to short binary values, which will be included as tags. + TX = TXWithoutTags#tx { tags = Tags }, + % Recursively turn the remaining data items into tx records. + DataItems = maps:from_list(lists:map( + fun({Key, Value}) -> + {Key, message_to_tx(Value)} + end, + RawDataItems + )), + % Set the data based on the remaining keys. + TXWithData = + case {TX#tx.data, maps:size(DataItems)} of + {Binary, 0} when is_binary(Binary) -> + TX; + {?DEFAULT_DATA, _} -> + TX#tx { data = DataItems }; + {Data, _} when is_map(Data) -> + TX#tx { data = maps:merge(Data, DataItems) }; + {Data, _} when is_record(Data, tx) -> + TX#tx { data = DataItems#{ data => Data } }; + {Data, _} when is_binary(Data) -> + TX#tx { data = DataItems#{ data => message_to_tx(Data) } } + end, + % ?event({prepared_tx_before_ids, + % {tags, {explicit, TXWithData#tx.tags}}, + % {data, TXWithData#tx.data} + % }), + ar_bundles:reset_ids(ar_bundles:normalize(TXWithData)); message_to_tx(Other) -> - ?event({unexpected_message_form, {explicit, Other}}), - throw(invalid_tx). + ?event({unexpected_message_form, {explicit, Other}}), + throw(invalid_tx). %% @doc Convert non-binary values to binary for serialization. decode_value(decimal, Value) -> - {item, Number, _} = hb_http_structured_fields:parse_item(Value), - Number; + {item, Number, _} = hb_http_structured_fields:parse_item(Value), + Number; decode_value(atom, Value) -> - {item, {string, AtomString}, _} = - hb_http_structured_fields:parse_item(Value), - binary_to_existing_atom(AtomString, latin1); + {item, {string, AtomString}, _} = + hb_http_structured_fields:parse_item(Value), + binary_to_existing_atom(AtomString, latin1); decode_value(OtherType, Value) -> - ?event({unexpected_type, OtherType, Value}), - throw({unexpected_type, OtherType, Value}). + ?event({unexpected_type, OtherType, Value}), + throw({unexpected_type, OtherType, Value}). %% @doc Convert a term to a typed key. to_typed_keys({Key, Value}) -> - to_typed_keys(Key, Value). + to_typed_keys(Key, Value). to_typed_keys(Key, Value) when is_binary(Value) -> - [{Key, Value}]; + [{Key, Value}]; to_typed_keys(Key, Value) when is_map(Value) -> - [{Key, message_to_tx(Value)}]; + [{Key, message_to_tx(Value)}]; to_typed_keys(Key, Value) -> - ItemKey = hb_pam:key_to_binary(Key), - {Type, BinaryValue} = encode_value(Value), - [ - {<<"Type:", ItemKey/binary>>, Type}, - {ItemKey, BinaryValue} - ]. + ItemKey = hb_pam:key_to_binary(Key), + {Type, BinaryValue} = encode_value(Value), + [ + {<<"Type:", ItemKey/binary>>, Type}, + {ItemKey, BinaryValue} + ]. %% @doc Convert a term to a binary representation, emitting its type for %% serialization as a separate tag. encode_value(Value) when is_integer(Value) -> - ?no_prod("Non-standardized type conversion invoked."), - [Encoded, _] = hb_http_structured_fields:item({item, Value, []}), - {<<"decimal">>, Encoded}; + ?no_prod("Non-standardized type conversion invoked."), + [Encoded, _] = hb_http_structured_fields:item({item, Value, []}), + {<<"decimal">>, Encoded}; encode_value(Value) when is_atom(Value) -> - ?no_prod("Non-standardized type conversion invoked."), - [EncodedIOList, _] = - hb_http_structured_fields:item( - {item, {string, atom_to_binary(Value, latin1)}, []}), - Encoded = list_to_binary(EncodedIOList), - {<<"atom">>, Encoded}; + ?no_prod("Non-standardized type conversion invoked."), + [EncodedIOList, _] = + hb_http_structured_fields:item( + {item, {string, atom_to_binary(Value, latin1)}, []}), + Encoded = list_to_binary(EncodedIOList), + {<<"atom">>, Encoded}; encode_value(Value) -> - Value. + Value. %% @doc Convert a #tx record into a message map recursively. tx_to_message(Binary) when is_binary(Binary) -> Binary; tx_to_message(TX) when is_record(TX, tx) -> - case lists:keyfind(<<"PAM-Large-Binary">>, 1, TX#tx.tags) of - false -> - do_tx_to_message(TX); - {_, _Size} -> - TX#tx.data - end. + case lists:keyfind(<<"PAM-Large-Binary">>, 1, TX#tx.tags) of + false -> + do_tx_to_message(TX); + {_, _Size} -> + TX#tx.data + end. do_tx_to_message(RawTX) -> - % Ensure the TX is fully deserialized. - TX = ar_bundles:deserialize(ar_bundles:normalize(RawTX)), - % We need to generate a map from each field of the tx record. - % Get the raw fields and values of the tx record and pair them. - Fields = record_info(fields, tx), - Values = tl(tuple_to_list(TX)), - TXKeyVals = lists:zip(Fields, Values), - % Convert the list of key-value pairs into a map. - UnfilteredTXMap = maps:from_list(TXKeyVals), - TXMap = maps:with(?TX_KEYS, UnfilteredTXMap), - {TXTagsUnparsed, FilteredTags} = lists:partition( - fun({Key, _}) -> - not lists:any( - fun(FilterKey) -> - case binary:longest_common_prefix([Key, FilterKey]) of - Length when Length == byte_size(FilterKey) -> - true; - _ -> - false - end - end, - ?FILTERED_TAGS - ); - (_) -> true - end, - TX#tx.tags - ), - % Parse tags that have a "Type:" prefix. - TagTypes = - [ - {Name, binary_to_existing_atom(Value, latin1)} - || - {<<"Type:", Name/binary>>, Value} <- FilteredTags - ], - TXTags = - lists:map( - fun({Name, BinaryValue}) -> - case lists:keyfind(Name, 1, TagTypes) of - false -> {Name, BinaryValue}; - {_, Type} -> {Name, decode_value(Type, BinaryValue)} - end - end, - TXTagsUnparsed - ), - % Next, merge the tags into the map. - MapWithoutData = maps:merge(TXMap, maps:from_list(TXTags)), - % Finally, merge the data into the map. - normalize( - case TX#tx.data of - Data when is_map(Data) -> - % If the data is a map, we need to recursively turn its children - % into messages from their tx representations. - ?event({merging_map_and_data, MapWithoutData, Data}), - maps:merge( - MapWithoutData, - maps:map( - fun(_, InnerValue) -> - tx_to_message(InnerValue) - end, - Data - ) - ); - Data when Data == ?DEFAULT_DATA -> - MapWithoutData; - Data when is_binary(Data) -> - MapWithoutData#{ data => Data }; - Data -> - ?event({unexpected_data_type, {explicit, Data}}), - ?event({was_processing, {explicit, TX}}), - throw(invalid_tx) - end - ). + % Ensure the TX is fully deserialized. + TX = ar_bundles:deserialize(ar_bundles:normalize(RawTX)), + % We need to generate a map from each field of the tx record. + % Get the raw fields and values of the tx record and pair them. + Fields = record_info(fields, tx), + Values = tl(tuple_to_list(TX)), + TXKeyVals = lists:zip(Fields, Values), + % Convert the list of key-value pairs into a map. + UnfilteredTXMap = maps:from_list(TXKeyVals), + TXMap = maps:with(?TX_KEYS, UnfilteredTXMap), + {TXTagsUnparsed, FilteredTags} = lists:partition( + fun({Key, _}) -> + not lists:any( + fun(FilterKey) -> + case binary:longest_common_prefix([Key, FilterKey]) of + Length when Length == byte_size(FilterKey) -> + true; + _ -> + false + end + end, + ?FILTERED_TAGS + ); + (_) -> true + end, + TX#tx.tags + ), + % Parse tags that have a "Type:" prefix. + TagTypes = + [ + {Name, binary_to_existing_atom(Value, latin1)} + || + {<<"Type:", Name/binary>>, Value} <- FilteredTags + ], + TXTags = + lists:map( + fun({Name, BinaryValue}) -> + case lists:keyfind(Name, 1, TagTypes) of + false -> {Name, BinaryValue}; + {_, Type} -> {Name, decode_value(Type, BinaryValue)} + end + end, + TXTagsUnparsed + ), + % Next, merge the tags into the map. + MapWithoutData = maps:merge(TXMap, maps:from_list(TXTags)), + % Finally, merge the data into the map. + normalize( + case TX#tx.data of + Data when is_map(Data) -> + % If the data is a map, we need to recursively turn its children + % into messages from their tx representations. + ?event({merging_map_and_data, MapWithoutData, Data}), + maps:merge( + MapWithoutData, + maps:map( + fun(_, InnerValue) -> + tx_to_message(InnerValue) + end, + Data + ) + ); + Data when Data == ?DEFAULT_DATA -> + MapWithoutData; + Data when is_binary(Data) -> + MapWithoutData#{ data => Data }; + Data -> + ?event({unexpected_data_type, {explicit, Data}}), + ?event({was_processing, {explicit, TX}}), + throw(invalid_tx) + end + ). %%% Tests basic_map_to_tx_test() -> - Msg = #{ normal_key => <<"NORMAL_VALUE">> }, - TX = message_to_tx(Msg), - ?assertEqual([{<<"normal_key">>, <<"NORMAL_VALUE">>}], TX#tx.tags). + Msg = #{ normal_key => <<"NORMAL_VALUE">> }, + TX = message_to_tx(Msg), + ?assertEqual([{<<"normal_key">>, <<"NORMAL_VALUE">>}], TX#tx.tags). %% @doc Test that the filter_default_tx_keys/1 function removes TX fields %% that have the default values found in the tx record, but not those that %% have been set by the user. default_tx_keys_removed_test() -> - TX = #tx { unsigned_id = << 1:256 >>, last_tx = << 2:256 >> }, - TXMap = #{ - unsigned_id => TX#tx.unsigned_id, - last_tx => TX#tx.last_tx, - <<"owner">> => TX#tx.owner, - <<"target">> => TX#tx.target, - data => TX#tx.data - }, - FilteredMap = filter_default_tx_keys(TXMap), - ?assertEqual(<< 1:256 >>, maps:get(unsigned_id, FilteredMap)), - ?assertEqual(<< 2:256 >>, maps:get(last_tx, FilteredMap, not_found)), - ?assertEqual(not_found, maps:get(<<"owner">>, FilteredMap, not_found)), - ?assertEqual(not_found, maps:get(<<"target">>, FilteredMap, not_found)). + TX = #tx { unsigned_id = << 1:256 >>, last_tx = << 2:256 >> }, + TXMap = #{ + unsigned_id => TX#tx.unsigned_id, + last_tx => TX#tx.last_tx, + <<"owner">> => TX#tx.owner, + <<"target">> => TX#tx.target, + data => TX#tx.data + }, + FilteredMap = filter_default_tx_keys(TXMap), + ?assertEqual(<< 1:256 >>, maps:get(unsigned_id, FilteredMap)), + ?assertEqual(<< 2:256 >>, maps:get(last_tx, FilteredMap, not_found)), + ?assertEqual(not_found, maps:get(<<"owner">>, FilteredMap, not_found)), + ?assertEqual(not_found, maps:get(<<"target">>, FilteredMap, not_found)). minimization_test() -> - Msg = #{ - unsigned_id => << 1:256 >>, - <<"id">> => << 2:256 >> - }, - MinimizedMsg = minimize(Msg), - ?event({minimized, MinimizedMsg}), - ?assertEqual(0, maps:size(MinimizedMsg)). + Msg = #{ + unsigned_id => << 1:256 >>, + <<"id">> => << 2:256 >> + }, + MinimizedMsg = minimize(Msg), + ?event({minimized, MinimizedMsg}), + ?assertEqual(0, maps:size(MinimizedMsg)). %% @doc Test that we can convert a message into a tx record and back. single_layer_message_to_tx_test() -> - Msg = #{ - last_tx => << 2:256 >>, - owner => << 3:4096 >>, - target => << 4:256 >>, - data => <<"DATA">>, - <<"Special-Key">> => <<"SPECIAL_VALUE">> - }, - TX = message_to_tx(Msg), - ?event({tx_to_message, {msg, Msg}, {tx, TX}}), - ?assertEqual(maps:get(last_tx, Msg), TX#tx.last_tx), - ?assertEqual(maps:get(owner, Msg), TX#tx.owner), - ?assertEqual(maps:get(target, Msg), TX#tx.target), - ?assertEqual(maps:get(data, Msg), TX#tx.data), - ?assertEqual({<<"Special-Key">>, <<"SPECIAL_VALUE">>}, - lists:keyfind(<<"Special-Key">>, 1, TX#tx.tags)). + Msg = #{ + last_tx => << 2:256 >>, + owner => << 3:4096 >>, + target => << 4:256 >>, + data => <<"DATA">>, + <<"Special-Key">> => <<"SPECIAL_VALUE">> + }, + TX = message_to_tx(Msg), + ?event({tx_to_message, {msg, Msg}, {tx, TX}}), + ?assertEqual(maps:get(last_tx, Msg), TX#tx.last_tx), + ?assertEqual(maps:get(owner, Msg), TX#tx.owner), + ?assertEqual(maps:get(target, Msg), TX#tx.target), + ?assertEqual(maps:get(data, Msg), TX#tx.data), + ?assertEqual({<<"Special-Key">>, <<"SPECIAL_VALUE">>}, + lists:keyfind(<<"Special-Key">>, 1, TX#tx.tags)). %% @doc Test that we can convert a #tx record into a message correctly. single_layer_tx_to_message_test() -> - TX = #tx { - unsigned_id = << 1:256 >>, - last_tx = << 2:256 >>, - owner = << 3:256 >>, - target = << 4:256 >>, - data = <<"DATA">>, - tags = [{<<"special_key">>, <<"SPECIAL_KEY">>}] - }, - Msg = tx_to_message(TX), - ?assertEqual(maps:get(<<"special_key">>, Msg), <<"SPECIAL_KEY">>), - ?assertEqual(<< "DATA">>, maps:get(<<"data">>, Msg)), - ?assertEqual(<< 2:256 >>, maps:get(<<"last_tx">>, Msg)), - ?assertEqual(<< 3:256 >>, maps:get(<<"owner">>, Msg)), - ?assertEqual(<< 4:256 >>, maps:get(<<"target">>, Msg)). + TX = #tx { + unsigned_id = << 1:256 >>, + last_tx = << 2:256 >>, + owner = << 3:256 >>, + target = << 4:256 >>, + data = <<"DATA">>, + tags = [{<<"special_key">>, <<"SPECIAL_KEY">>}] + }, + Msg = tx_to_message(TX), + ?assertEqual(maps:get(<<"special_key">>, Msg), <<"SPECIAL_KEY">>), + ?assertEqual(<< "DATA">>, maps:get(<<"data">>, Msg)), + ?assertEqual(<< 2:256 >>, maps:get(<<"last_tx">>, Msg)), + ?assertEqual(<< 3:256 >>, maps:get(<<"owner">>, Msg)), + ?assertEqual(<< 4:256 >>, maps:get(<<"target">>, Msg)). %% @doc Test that the message matching function works. match_test() -> - Msg = #{ a => 1, b => 2 }, - TX = message_to_tx(Msg), - Msg2 = tx_to_message(TX), - ?assert(match(Msg, Msg2)). + Msg = #{ a => 1, b => 2 }, + TX = message_to_tx(Msg), + Msg2 = tx_to_message(TX), + ?assert(match(Msg, Msg2)). %% @doc Test that two txs match. Note: This function uses tx_to_message/1 %% underneath, which (depending on the test) could potentially lead to false %% positives. txs_match(TX1, TX2) -> - match(tx_to_message(TX1), tx_to_message(TX2)). + match(tx_to_message(TX1), tx_to_message(TX2)). %% @doc Structured field parsing tests. structured_field_atom_parsing_test() -> - Msg = #{ highly_unusual_http_header => highly_unusual_value }, - ?assert(match(Msg, tx_to_message(message_to_tx(Msg)))). + Msg = #{ highly_unusual_http_header => highly_unusual_value }, + ?assert(match(Msg, tx_to_message(message_to_tx(Msg)))). structured_field_decimal_parsing_test() -> - Msg = #{ integer_field => 1234567890 }, - ?assert(match(Msg, tx_to_message(message_to_tx(Msg)))). + Msg = #{ integer_field => 1234567890 }, + ?assert(match(Msg, tx_to_message(message_to_tx(Msg)))). binary_to_binary_test() -> - % Serialization must be able to turn a raw binary into a TX, then turn - % that TX back into a binary and have the result match the original. - Bin = <<"THIS IS A BINARY, NOT A NORMAL MESSAGE">>, - Msg = message_to_tx(Bin), - ?assertEqual(Bin, tx_to_message(Msg)). + % Serialization must be able to turn a raw binary into a TX, then turn + % that TX back into a binary and have the result match the original. + Bin = <<"THIS IS A BINARY, NOT A NORMAL MESSAGE">>, + Msg = message_to_tx(Bin), + ?assertEqual(Bin, tx_to_message(Msg)). %% @doc Test that the data field is correctly managed when we have multiple %% uses for it (the 'data' key itself, as well as keys that cannot fit in %% tags). message_with_large_keys_test() -> - Msg = #{ - <<"normal_key">> => <<"normal_value">>, - <<"large_key">> => << 0:((1 + ?MAX_TAG_VAL) * 8) >>, - <<"another_large_key">> => << 0:((1 + ?MAX_TAG_VAL) * 8) >>, - <<"another_normal_key">> => <<"another_normal_value">> - }, - ?assert(match(Msg, tx_to_message(message_to_tx(Msg)))). + Msg = #{ + <<"normal_key">> => <<"normal_value">>, + <<"large_key">> => << 0:((1 + ?MAX_TAG_VAL) * 8) >>, + <<"another_large_key">> => << 0:((1 + ?MAX_TAG_VAL) * 8) >>, + <<"another_normal_key">> => <<"another_normal_value">> + }, + ?assert(match(Msg, tx_to_message(message_to_tx(Msg)))). %% @doc Check that large keys and data fields are correctly handled together. nested_message_with_large_keys_and_data_test() -> - Msg = #{ - <<"normal_key">> => <<"normal_value">>, - <<"large_key">> => << 0:(?MAX_TAG_VAL * 16) >>, - <<"another_large_key">> => << 0:(?MAX_TAG_VAL * 16) >>, - <<"another_normal_key">> => <<"another_normal_value">>, - data => <<"Hey from the data field!">> - }, - TX = message_to_tx(Msg), - Msg2 = tx_to_message(TX), - ?event({matching, {input, Msg}, {tx, TX}, {output, Msg2}}), - ?assert(match(Msg, Msg2)). + Msg = #{ + <<"normal_key">> => <<"normal_value">>, + <<"large_key">> => << 0:(?MAX_TAG_VAL * 16) >>, + <<"another_large_key">> => << 0:(?MAX_TAG_VAL * 16) >>, + <<"another_normal_key">> => <<"another_normal_value">>, + data => <<"Hey from the data field!">> + }, + TX = message_to_tx(Msg), + Msg2 = tx_to_message(TX), + ?event({matching, {input, Msg}, {tx, TX}, {output, Msg2}}), + ?assert(match(Msg, Msg2)). simple_nested_message_test() -> - Msg = #{ - a => <<"1">>, - nested => #{ <<"b">> => <<"1">> }, - c => <<"3">> - }, - TX = message_to_tx(Msg), - Msg2 = tx_to_message(TX), - ?event({matching, {input, Msg}, {output, Msg2}}), - ?assert( - match( - Msg, - Msg2 - ) - ). + Msg = #{ + a => <<"1">>, + nested => #{ <<"b">> => <<"1">> }, + c => <<"3">> + }, + TX = message_to_tx(Msg), + Msg2 = tx_to_message(TX), + ?event({matching, {input, Msg}, {output, Msg2}}), + ?assert( + match( + Msg, + Msg2 + ) + ). %% @doc Test that the data field is correctly managed when we have multiple %% uses for it (the 'data' key itself, as well as keys that cannot fit in %% tags). nested_message_with_large_data_test() -> - Msg = #{ - <<"tx_depth">> => <<"outer">>, - data => #{ - <<"tx_map_item">> => - #{ - <<"tx_depth">> => <<"inner">>, - <<"large_data_inner">> => << 0:((1 + ?MAX_TAG_VAL) * 8) >> - }, - <<"large_data_outer">> => << 0:((1 + ?MAX_TAG_VAL) * 8) >> - } - }, - ?assert(match(Msg, tx_to_message(message_to_tx(Msg)))). + Msg = #{ + <<"tx_depth">> => <<"outer">>, + data => #{ + <<"tx_map_item">> => + #{ + <<"tx_depth">> => <<"inner">>, + <<"large_data_inner">> => << 0:((1 + ?MAX_TAG_VAL) * 8) >> + }, + <<"large_data_outer">> => << 0:((1 + ?MAX_TAG_VAL) * 8) >> + } + }, + ?assert(match(Msg, tx_to_message(message_to_tx(Msg)))). %% @doc Test that we can convert a 3 layer nested message into a tx record and back. deeply_nested_message_with_data_test() -> - Msg = #{ - <<"tx_depth">> => <<"outer">>, - data => #{ - <<"tx_map_item">> => - #{ - <<"tx_depth">> => <<"inner">>, - data => #{ - <<"tx_depth">> => <<"innermost">>, - data => <<"DATA">> - } - } - } - }, - ?assert(match(Msg, tx_to_message(message_to_tx(Msg)))). + Msg = #{ + <<"tx_depth">> => <<"outer">>, + data => #{ + <<"tx_map_item">> => + #{ + <<"tx_depth">> => <<"inner">>, + data => #{ + <<"tx_depth">> => <<"innermost">>, + data => <<"DATA">> + } + } + } + }, + ?assert(match(Msg, tx_to_message(message_to_tx(Msg)))). nested_structured_fields_test() -> - NestedMsg = #{ a => #{ b => 1 } }, - ?assert( - match( - NestedMsg, - tx_to_message(message_to_tx(NestedMsg)) - ) - ). + NestedMsg = #{ a => #{ b => 1 } }, + ?assert( + match( + NestedMsg, + tx_to_message(message_to_tx(NestedMsg)) + ) + ). nested_message_with_large_keys_test() -> - Msg = #{ - a => <<"1">>, - long_data => << 0:((1 + ?MAX_TAG_VAL) * 8) >>, - nested => #{ <<"b">> => <<"1">> }, - c => <<"3">> - }, - ResMsg = tx_to_message(message_to_tx(Msg)), - ?event({matching, {input, Msg}, {output, ResMsg}}), - ?assert(match(Msg, ResMsg)). + Msg = #{ + a => <<"1">>, + long_data => << 0:((1 + ?MAX_TAG_VAL) * 8) >>, + nested => #{ <<"b">> => <<"1">> }, + c => <<"3">> + }, + ResMsg = tx_to_message(message_to_tx(Msg)), + ?event({matching, {input, Msg}, {output, ResMsg}}), + ?assert(match(Msg, ResMsg)). %% @doc Test that we can convert a signed tx into a message and back. signed_tx_to_message_and_back_test() -> - TX = #tx { - data = <<"TEST_DATA">>, - tags = [{<<"TEST_KEY">>, <<"TEST_VALUE">>}] - }, - SignedTX = ar_bundles:sign_item(TX, hb:wallet()), - ?assert(ar_bundles:verify_item(SignedTX)), - SignedMsg = tx_to_message(SignedTX), - SignedTX2 = message_to_tx(SignedMsg), - ?assert(ar_bundles:verify_item(SignedTX2)). + TX = #tx { + data = <<"TEST_DATA">>, + tags = [{<<"TEST_KEY">>, <<"TEST_VALUE">>}] + }, + SignedTX = ar_bundles:sign_item(TX, hb:wallet()), + ?assert(ar_bundles:verify_item(SignedTX)), + SignedMsg = tx_to_message(SignedTX), + SignedTX2 = message_to_tx(SignedMsg), + ?assert(ar_bundles:verify_item(SignedTX2)). signed_deep_tx_serialize_and_deserialize_test() -> - TX = #tx { - tags = [{<<"TEST_KEY">>, <<"TEST_VALUE">>}], - data = #{ - <<"NESTED_TX">> => - #tx { - data = <<"NESTED_DATA">>, - tags = [{<<"NESTED_KEY">>, <<"NESTED_VALUE">>}] - } - } - }, - SignedTX = ar_bundles:deserialize( - ar_bundles:sign_item(TX, hb:wallet()) - ), - ?assert(ar_bundles:verify_item(SignedTX)), - SerializedTX = serialize(SignedTX), - DeserializedTX = deserialize(SerializedTX), - ?assert( - match( - tx_to_message(SignedTX), - DeserializedTX - ) - ). + TX = #tx { + tags = [{<<"TEST_KEY">>, <<"TEST_VALUE">>}], + data = #{ + <<"NESTED_TX">> => + #tx { + data = <<"NESTED_DATA">>, + tags = [{<<"NESTED_KEY">>, <<"NESTED_VALUE">>}] + } + } + }, + SignedTX = ar_bundles:deserialize( + ar_bundles:sign_item(TX, hb:wallet()) + ), + ?assert(ar_bundles:verify_item(SignedTX)), + SerializedTX = serialize(SignedTX), + DeserializedTX = deserialize(SerializedTX), + ?assert( + match( + tx_to_message(SignedTX), + DeserializedTX + ) + ). calculate_unsigned_message_id_test() -> - Msg = #{ - data => <<"DATA">>, - <<"special_key">> => <<"SPECIAL_KEY">> - }, - UnsignedTX = message_to_tx(Msg), - UnsignedMessage = tx_to_message(UnsignedTX), - ?assertEqual( - hb_util:encode(ar_bundles:id(UnsignedTX, unsigned)), - hb_util:id(UnsignedMessage, unsigned) - ). + Msg = #{ + data => <<"DATA">>, + <<"special_key">> => <<"SPECIAL_KEY">> + }, + UnsignedTX = message_to_tx(Msg), + UnsignedMessage = tx_to_message(UnsignedTX), + ?assertEqual( + hb_util:encode(ar_bundles:id(UnsignedTX, unsigned)), + hb_util:id(UnsignedMessage, unsigned) + ). sign_serialize_deserialize_verify_test() -> - Msg = #{ - data => <<"DATA">>, - <<"special_key">> => <<"SPECIAL_KEY">> - }, - TX = message_to_tx(Msg), - SignedTX = ar_bundles:sign_item(TX, hb:wallet()), - ?assert(ar_bundles:verify_item(SignedTX)), - SerializedMsg = serialize(SignedTX), - DeserializedMsg = deserialize(SerializedMsg), - DeserializedTX = message_to_tx(DeserializedMsg), - ?assert(ar_bundles:verify_item(DeserializedTX)). + Msg = #{ + data => <<"DATA">>, + <<"special_key">> => <<"SPECIAL_KEY">> + }, + TX = message_to_tx(Msg), + SignedTX = ar_bundles:sign_item(TX, hb:wallet()), + ?assert(ar_bundles:verify_item(SignedTX)), + SerializedMsg = serialize(SignedTX), + DeserializedMsg = deserialize(SerializedMsg), + DeserializedTX = message_to_tx(DeserializedMsg), + ?assert(ar_bundles:verify_item(DeserializedTX)). unsigned_id_test() -> - UnsignedTX = ar_bundles:normalize(#tx { data = <<"TEST_DATA">> }), - UnsignedMessage = tx_to_message(UnsignedTX), - ?assertEqual( - hb_util:encode(ar_bundles:id(UnsignedTX, unsigned)), - hb_util:id(UnsignedMessage, unsigned) - ). + UnsignedTX = ar_bundles:normalize(#tx { data = <<"TEST_DATA">> }), + UnsignedMessage = tx_to_message(UnsignedTX), + ?assertEqual( + hb_util:encode(ar_bundles:id(UnsignedTX, unsigned)), + hb_util:id(UnsignedMessage, unsigned) + ). signed_id_test_disabled() -> - TX = #tx { - data = <<"TEST_DATA">>, - tags = [{<<"TEST_KEY">>, <<"TEST_VALUE">>}] - }, - SignedTX = ar_bundles:sign_item(TX, hb:wallet()), - ?assert(ar_bundles:verify_item(SignedTX)), - SignedMsg = tx_to_message(SignedTX), - ?assertEqual( - hb_util:encode(ar_bundles:id(SignedTX, signed)), - hb_util:id(SignedMsg, signed) - ). \ No newline at end of file + TX = #tx { + data = <<"TEST_DATA">>, + tags = [{<<"TEST_KEY">>, <<"TEST_VALUE">>}] + }, + SignedTX = ar_bundles:sign_item(TX, hb:wallet()), + ?assert(ar_bundles:verify_item(SignedTX)), + SignedMsg = tx_to_message(SignedTX), + ?assertEqual( + hb_util:encode(ar_bundles:id(SignedTX, signed)), + hb_util:id(SignedMsg, signed) + ). \ No newline at end of file diff --git a/src/hb_metrics_collector.erl b/src/hb_metrics_collector.erl index 8f804fa0..f98add90 100644 --- a/src/hb_metrics_collector.erl +++ b/src/hb_metrics_collector.erl @@ -1,11 +1,11 @@ -module(hb_metrics_collector). -export( - [ - deregister_cleanup/1, - collect_mf/2, - collect_metrics/2 - ] + [ + deregister_cleanup/1, + collect_mf/2, + collect_metrics/2 + ] ). -behaviour(prometheus_collector). %%==================================================================== @@ -14,49 +14,49 @@ deregister_cleanup(_) -> ok. collect_mf(_Registry, Callback) -> - {Uptime, _} = erlang:statistics(wall_clock), - Callback( - create_gauge( - process_uptime_seconds, - "The number of seconds the Erlang process has been up.", - Uptime - ) - ), + {Uptime, _} = erlang:statistics(wall_clock), + Callback( + create_gauge( + process_uptime_seconds, + "The number of seconds the Erlang process has been up.", + Uptime + ) + ), - SystemLoad = cpu_sup:avg5(), + SystemLoad = cpu_sup:avg5(), - Callback( - create_gauge( - system_load, - "The load values are proportional to how long" - " time a runnable Unix process has to spend in the run queue" - " before it is scheduled. Accordingly, higher values mean" - " more system load", - SystemLoad - ) - ), + Callback( + create_gauge( + system_load, + "The load values are proportional to how long" + " time a runnable Unix process has to spend in the run queue" + " before it is scheduled. Accordingly, higher values mean" + " more system load", + SystemLoad + ) + ), - ok. + ok. collect_metrics(system_load, SystemLoad) -> - %% Return the gauge metric with no labels - prometheus_model_helpers:gauge_metrics( - [ - {[], SystemLoad} - ] - ); + %% Return the gauge metric with no labels + prometheus_model_helpers:gauge_metrics( + [ + {[], SystemLoad} + ] + ); collect_metrics(process_uptime_seconds, Uptime) -> - %% Convert the uptime from milliseconds to seconds - UptimeSeconds = Uptime / 1000, + %% Convert the uptime from milliseconds to seconds + UptimeSeconds = Uptime / 1000, - %% Return the gauge metric with no labels - prometheus_model_helpers:gauge_metrics( - [ - {[], UptimeSeconds} - ] - ). + %% Return the gauge metric with no labels + prometheus_model_helpers:gauge_metrics( + [ + {[], UptimeSeconds} + ] + ). %%==================================================================== %% Private Parts %%==================================================================== create_gauge(Name, Help, Data) -> - prometheus_model_helpers:create_mf(Name, Help, gauge, ?MODULE, Data). + prometheus_model_helpers:create_mf(Name, Help, gauge, ?MODULE, Data). \ No newline at end of file diff --git a/src/hb_opts.erl b/src/hb_opts.erl index 958c0437..7a69ff9a 100644 --- a/src/hb_opts.erl +++ b/src/hb_opts.erl @@ -25,17 +25,17 @@ config() -> %% that an assignment has been scheduled for a message. %% Options: aggressive(!), local_confirmation, remote_confirmation scheduling_mode => local_confirmation, - %% Compute mode: Determines whether the CU should attempt to execute - %% more messages on a process after it has returned a result. - %% Options: aggressive, lazy - compute_mode => lazy, - %% Choice of remote nodes for tasks that are not local to hyperbeam. + %% Compute mode: Determines whether the CU should attempt to execute + %% more messages on a process after it has returned a result. + %% Options: aggressive, lazy + compute_mode => lazy, + %% Choice of remote nodes for tasks that are not local to hyperbeam. http_host => "localhost", gateway => "https://arweave.net", bundler => "https://up.arweave.net", - %% Choice of nodes for remote tasks, in the form of a map between - %% node addresses and HTTP URLs. - %% `_` is a wildcard for any other address that is not specified. + %% Choice of nodes for remote tasks, in the form of a map between + %% node addresses and HTTP URLs. + %% `_` is a wildcard for any other address that is not specified. nodes => #{ compute => #{ @@ -54,16 +54,16 @@ config() -> '_' => "http://localhost:8734" } }, - %% Location of the wallet keyfile on disk that this node will use. + %% Location of the wallet keyfile on disk that this node will use. key_location => "hyperbeam-key.json", - %% Default page limit for pagination of results from the APIs. - %% Currently used in the SU devices. + %% Default page limit for pagination of results from the APIs. + %% Currently used in the SU devices. default_page_limit => 5, - %% The time-to-live that should be specified when we register - %% ourselves as a scheduler on the network. + %% The time-to-live that should be specified when we register + %% ourselves as a scheduler on the network. scheduler_location_ttl => 60 * 60 * 24 * 30, - %% Preloaded devices for the node to use. These names override - %% resolution of devices via ID to the default implementations. + %% Preloaded devices for the node to use. These names override + %% resolution of devices via ID to the default implementations. preloaded_devices => #{ <<"Stack">> => dev_stack, @@ -80,33 +80,33 @@ config() -> <<"Compute">> => dev_cu, <<"P4">> => dev_p4 }, - %% The stacks of devices that the node should expose by default. - %% These represent the core flows of functionality of the node. + %% The stacks of devices that the node should expose by default. + %% These represent the core flows of functionality of the node. default_device_stacks => [ {<<"data">>, {<<"read">>, [dev_p4, dev_lookup]}}, {<<"su">>, {<<"schedule">>, [dev_p4, dev_scheduler]}}, {<<"cu">>, {<<"execute">>, [dev_p4, dev_cu]}} ], - %% Should the node attempt to access data from remote caches for - %% client requests? - access_remote_cache_for_client => false, - %% Should the node attempt to load devices from remote signers? - load_remote_devices => false, - %% The list of device signers that the node should trust. - trusted_device_signers => [], - %% What should the node do if a client error occurs? - client_error_strategy => throw, + %% Should the node attempt to access data from remote caches for + %% client requests? + access_remote_cache_for_client => false, + %% Should the node attempt to load devices from remote signers? + load_remote_devices => false, + %% The list of device signers that the node should trust. + trusted_device_signers => [], + %% What should the node do if a client error occurs? + client_error_strategy => throw, % Dev options local_store => [{hb_store_fs, #{ prefix => "TEST-data" }}], mode => debug, - debug_stack_depth => 20, - debug_print_map_line_threshold => 30, - debug_print_binary_max => 15, - debug_print_indent => 4, + debug_stack_depth => 20, + debug_print_map_line_threshold => 30, + debug_print_binary_max => 15, + debug_print_indent => 4, debug_print => false, - cache_results => false, - stack_print_prefixes => ["hb", "dev", "ar"] + cache_results => false, + stack_print_prefixes => ["hb", "dev", "ar"] }. %% @doc Get an option from the global options, optionally overriding with a @@ -119,30 +119,30 @@ config() -> get(Key) -> ?MODULE:get(Key, undefined). get(Key, Default) -> ?MODULE:get(Key, Default, #{}). get(Key, Default, Opts = #{ only := local }) -> - case maps:find(Key, Opts) of - {ok, Value} -> Value; - error -> - Default - end; + case maps:find(Key, Opts) of + {ok, Value} -> Value; + error -> + Default + end; get(Key, Default, #{ only := global }) -> - case global_get(Key, hb_opts_not_found) of - hb_opts_not_found -> Default; - Value -> Value - end; + case global_get(Key, hb_opts_not_found) of + hb_opts_not_found -> Default; + Value -> Value + end; get(Key, Default, Opts = #{ prefer := global }) -> - case ?MODULE:get(Key, hb_opts_not_found, #{ only => global }) of - hb_opts_not_found -> ?MODULE:get(Key, Default, Opts#{ only => local }); - Value -> Value - end; + case ?MODULE:get(Key, hb_opts_not_found, #{ only => global }) of + hb_opts_not_found -> ?MODULE:get(Key, Default, Opts#{ only => local }); + Value -> Value + end; get(Key, Default, Opts = #{ prefer := local }) -> - case ?MODULE:get(Key, hb_opts_not_found, Opts#{ only => local }) of - hb_opts_not_found -> - ?MODULE:get(Key, Default, Opts#{ only => global }); - Value -> Value - end; + case ?MODULE:get(Key, hb_opts_not_found, Opts#{ only => local }) of + hb_opts_not_found -> + ?MODULE:get(Key, Default, Opts#{ only => global }); + Value -> Value + end; get(Key, Default, Opts) -> - % No preference was set in Opts, so we default to local. - ?MODULE:get(Key, Default, Opts#{ prefer => local }). + % No preference was set in Opts, so we default to local. + ?MODULE:get(Key, Default, Opts#{ prefer => local }). -define(ENV_KEYS, #{ @@ -160,16 +160,16 @@ get(Key, Default, Opts) -> end, "TEST-data" }, - mode => - {"HB_MODE", fun list_to_existing_atom/1}, - debug_print => - {"HB_PRINT", - fun - (Str) when Str == "1" -> true; - (Str) when Str == "true" -> true; - (Str) -> string:tokens(Str, ",") - end - } + mode => + {"HB_MODE", fun list_to_existing_atom/1}, + debug_print => + {"HB_PRINT", + fun + (Str) when Str == "1" -> true; + (Str) when Str == "true" -> true; + (Str) -> string:tokens(Str, ",") + end + } } ). @@ -179,11 +179,11 @@ global_get(Key, Default) -> Default -> config_lookup(Key, Default); {EnvKey, ValParser, DefaultValue} when is_function(ValParser) -> ValParser(os:getenv(EnvKey, DefaultValue)); - {EnvKey, ValParser} when is_function(ValParser) -> - case os:getenv(EnvKey, not_found) of - not_found -> config_lookup(Key, Default); - Value -> ValParser(Value) - end; + {EnvKey, ValParser} when is_function(ValParser) -> + case os:getenv(EnvKey, not_found) of + not_found -> config_lookup(Key, Default); + Value -> ValParser(Value) + end; {EnvKey, DefaultValue} -> os:getenv(EnvKey, DefaultValue) end. @@ -198,28 +198,28 @@ config_lookup(Key, Default) -> maps:get(Key, config(), Default). global_get_test() -> ?assertEqual(debug, ?MODULE:get(mode)), ?assertEqual(debug, ?MODULE:get(mode, production)), - ?assertEqual(undefined, ?MODULE:get(unset_global_key)), - ?assertEqual(1234, ?MODULE:get(unset_global_key, 1234)). + ?assertEqual(undefined, ?MODULE:get(unset_global_key)), + ?assertEqual(1234, ?MODULE:get(unset_global_key, 1234)). local_get_test() -> - Local = #{ only => local }, - ?assertEqual(undefined, - ?MODULE:get(test_key, undefined, Local)), - ?assertEqual(correct, - ?MODULE:get(test_key, undefined, Local#{ test_key => correct })). + Local = #{ only => local }, + ?assertEqual(undefined, + ?MODULE:get(test_key, undefined, Local)), + ?assertEqual(correct, + ?MODULE:get(test_key, undefined, Local#{ test_key => correct })). local_preference_test() -> - Local = #{ prefer => local }, - ?assertEqual(correct, - ?MODULE:get(test_key, undefined, Local#{ test_key => correct })), - ?assertEqual(correct, - ?MODULE:get(mode, undefined, Local#{ mode => correct })), - ?assertNotEqual(undefined, - ?MODULE:get(mode, undefined, Local)). + Local = #{ prefer => local }, + ?assertEqual(correct, + ?MODULE:get(test_key, undefined, Local#{ test_key => correct })), + ?assertEqual(correct, + ?MODULE:get(mode, undefined, Local#{ mode => correct })), + ?assertNotEqual(undefined, + ?MODULE:get(mode, undefined, Local)). global_preference_test() -> - Global = #{ prefer => global }, - ?assertEqual(undefined, ?MODULE:get(test_key, undefined, Global)), - ?assertNotEqual(incorrect, - ?MODULE:get(mode, undefined, Global#{ mode => incorrect })), - ?assertNotEqual(undefined, ?MODULE:get(mode, undefined, Global)). \ No newline at end of file + Global = #{ prefer => global }, + ?assertEqual(undefined, ?MODULE:get(test_key, undefined, Global)), + ?assertNotEqual(incorrect, + ?MODULE:get(mode, undefined, Global#{ mode => incorrect })), + ?assertNotEqual(undefined, ?MODULE:get(mode, undefined, Global)). \ No newline at end of file diff --git a/src/hb_pam.erl b/src/hb_pam.erl index c2dfad06..9de3a063 100644 --- a/src/hb_pam.erl +++ b/src/hb_pam.erl @@ -63,67 +63,67 @@ %% keys of a map. 'Special' keys which do not exist as values in the message's %% map are simply ignored. resolve(Msg1, Request) -> - resolve(Msg1, Request, default_runtime_opts(Msg1)). + resolve(Msg1, Request, default_runtime_opts(Msg1)). resolve(Msg1, Msg2, Opts) when is_map(Msg2) and is_map(Opts) -> - prepare_resolve(Msg1, Msg2, Opts); + prepare_resolve(Msg1, Msg2, Opts); resolve(Msg1, Key, Opts) -> - prepare_resolve(Msg1, #{ path => Key }, Opts). + prepare_resolve(Msg1, #{ path => Key }, Opts). %% @doc Internal function for resolving the path from a message and loading %% the function to call. prepare_resolve(Msg1, Msg2, Opts) -> - %?event({pre_resolve_func_load, {msg1, Msg1}, {msg2, Msg2}, {opts, Opts}}), - {Fun, NewOpts} = - try - Key = hb_path:hd(Msg2, Opts), - % Try to load the device and get the function to call. - {Status, ReturnedFun} = message_to_fun(Msg1, Key), - % Next, add an option to the Opts map to indicate if we should - % add the key to the start of the arguments. Note: This option - % is used downstream by other devices (like `dev_stack`), so - % should be changed with care. - { - ReturnedFun, - Opts#{ - add_key => - case Status of - add_key -> Key; - _ -> false - end - } - } - catch - Class:Exception:Stacktrace -> - handle_error( - loading_device, - {Class, Exception, Stacktrace}, - Opts - ) - end, - do_resolve(Msg1, Fun, Msg2, NewOpts). + %?event({pre_resolve_func_load, {msg1, Msg1}, {msg2, Msg2}, {opts, Opts}}), + {Fun, NewOpts} = + try + Key = hb_path:hd(Msg2, Opts), + % Try to load the device and get the function to call. + {Status, ReturnedFun} = message_to_fun(Msg1, Key), + % Next, add an option to the Opts map to indicate if we should + % add the key to the start of the arguments. Note: This option + % is used downstream by other devices (like `dev_stack`), so + % should be changed with care. + { + ReturnedFun, + Opts#{ + add_key => + case Status of + add_key -> Key; + _ -> false + end + } + } + catch + Class:Exception:Stacktrace -> + handle_error( + loading_device, + {Class, Exception, Stacktrace}, + Opts + ) + end, + do_resolve(Msg1, Fun, Msg2, NewOpts). do_resolve(Msg1, Fun, Msg2, Opts) -> - ?event({resolving, Fun, {msg1, Msg1}, {msg2, Msg2}, {opts, Opts}}), - % First, determine the arguments to pass to the function. - % While calculating the arguments we unset the add_key option. - UserOpts = maps:remove(add_key, Opts), - Args = - case maps:get(add_key, Opts, false) of - false -> [Msg1, Msg2, UserOpts]; - Key -> [Key, Msg1, Msg2, UserOpts] - end, - % Then, try to execute the function. - try - Msg3 = apply(Fun, truncate_args(Fun, Args)), - ?event({resolved_result, {explicit, Msg3}}), - handle_resolved_result(Msg3, Msg2, UserOpts) - catch - ExecClass:ExecException:ExecStacktrace -> - handle_error( - device_call, - {ExecClass, ExecException, ExecStacktrace}, - Opts - ) - end. + ?event({resolving, Fun, {msg1, Msg1}, {msg2, Msg2}, {opts, Opts}}), + % First, determine the arguments to pass to the function. + % While calculating the arguments we unset the add_key option. + UserOpts = maps:remove(add_key, Opts), + Args = + case maps:get(add_key, Opts, false) of + false -> [Msg1, Msg2, UserOpts]; + Key -> [Key, Msg1, Msg2, UserOpts] + end, + % Then, try to execute the function. + try + Msg3 = apply(Fun, truncate_args(Fun, Args)), + ?event({resolved_result, {explicit, Msg3}}), + handle_resolved_result(Msg3, Msg2, UserOpts) + catch + ExecClass:ExecException:ExecStacktrace -> + handle_error( + device_call, + {ExecClass, ExecException, ExecStacktrace}, + Opts + ) + end. %% @doc Internal function for handling the result of a device call. %% If the result is a binary or something that doesn't have an `ok` status, @@ -136,63 +136,63 @@ do_resolve(Msg1, Fun, Msg2, Opts) -> %% If additional elements are included as part of the result, we pass them %% through to the caller. handle_resolved_result(Msg3, _Msg2, _UserOpts) when is_binary(Msg3) -> - Msg3; + Msg3; handle_resolved_result(Result, Msg2, Opts) when is_tuple(Result) -> - handle_resolved_result(tuple_to_list(Result), Msg2, Opts); + handle_resolved_result(tuple_to_list(Result), Msg2, Opts); handle_resolved_result(Msg2List = [Status, Result|_], _Msg2, _Opts) - when Status =/= ok orelse not is_map(Result) -> - list_to_tuple(Msg2List); + when Status =/= ok orelse not is_map(Result) -> + list_to_tuple(Msg2List); handle_resolved_result([ok, Msg3Raw | Rest], Msg2, Opts) -> - Msg3 = - case hb_opts:get(update_hashpath, true, Opts#{ only => local }) of - true -> - % ?event( - % {pushing_hashpath_onto, - % {msg3, {explicit, Msg3Raw}}, - % {msg2, Msg2}, - % {opts, Opts} - % } - % ), - hb_path:push(hashpath, Msg3Raw, Msg2); - false -> Msg3Raw - end, - case hb_opts:get(cache_results, true, Opts) of - true -> - hb_cache:write( - hb_opts:get(store, no_valid_store, Opts), - hb_message:message_to_tx(Msg3) - ); - false -> - ok - end, - case hb_path:tl(Msg2, Opts) of - Res when Res == undefined orelse Res == [] -> - % The path resolved to the last element, so we return - % to the caller. - list_to_tuple([ok, Msg3 | Rest]); - NextMsg -> - % There are more elements in the path, so we recurse. - ?event({recursing, {input, Msg3}, {next, NextMsg}}), - resolve(Msg3, NextMsg, Opts) - end. + Msg3 = + case hb_opts:get(update_hashpath, true, Opts#{ only => local }) of + true -> + % ?event( + % {pushing_hashpath_onto, + % {msg3, {explicit, Msg3Raw}}, + % {msg2, Msg2}, + % {opts, Opts} + % } + % ), + hb_path:push(hashpath, Msg3Raw, Msg2); + false -> Msg3Raw + end, + case hb_opts:get(cache_results, true, Opts) of + true -> + hb_cache:write( + hb_opts:get(store, no_valid_store, Opts), + hb_message:message_to_tx(Msg3) + ); + false -> + ok + end, + case hb_path:tl(Msg2, Opts) of + Res when Res == undefined orelse Res == [] -> + % The path resolved to the last element, so we return + % to the caller. + list_to_tuple([ok, Msg3 | Rest]); + NextMsg -> + % There are more elements in the path, so we recurse. + ?event({recursing, {input, Msg3}, {next, NextMsg}}), + resolve(Msg3, NextMsg, Opts) + end. %% @doc Shortcut for resolving a key in a message without its %% status if it is `ok`. This makes it easier to write complex %% logic on top of messages while maintaining a functional style. get(Path, Msg) -> - get(Path, Msg, default_runtime_opts(Msg)). + get(Path, Msg, default_runtime_opts(Msg)). get(Path, Msg, Opts) -> - ensure_ok(Path, resolve(Msg, #{ path => Path }, Opts), Opts). + ensure_ok(Path, resolve(Msg, #{ path => Path }, Opts), Opts). %% @doc Get the value of a key from a message, returning a default value if the %% key is not found. get_default(Key, Msg, Default) -> - get_default(Msg, Key, Default, #{}). + get_default(Msg, Key, Default, #{}). get_default(Key, Msg, Default, Opts) -> - case resolve(Msg, Key, Opts) of - {ok, Value} -> Value; - {error, _} -> Default - end. + case resolve(Msg, Key, Opts) of + {ok, Value} -> Value; + {error, _} -> Default + end. %% @doc Shortcut to get the list of keys from a message. keys(Msg) -> keys(Msg, #{}). @@ -203,66 +203,66 @@ keys(Msg, Opts) -> get(keys, Msg, Opts). %% `set` works with maps and recursive paths while maintaining the appropriate %% `HashPath` for each step. set(Msg1, Msg2) -> - set(Msg1, Msg2, #{}). + set(Msg1, Msg2, #{}). set(Msg1, Msg2, Opts) when is_map(Msg2) -> - ?event({set_called, {msg1, Msg1}, {msg2, Msg2}}), - case ?IS_EMPTY_MESSAGE(Msg2) of - true -> Msg1; - false -> - % First, get the first key and value to set. - Key = hd(keys(Msg2, Opts#{ hashpath => ignore })), - Val = get(Key, Msg2, Opts), - ?event({got_val_to_set, {key, Key}, {val, Val}}), - % Then, set the key and recurse, removing the key from the Msg2. - set(set(Msg1, Key, Val, Opts), remove(Msg2, Key, Opts), Opts) - end. + ?event({set_called, {msg1, Msg1}, {msg2, Msg2}}), + case ?IS_EMPTY_MESSAGE(Msg2) of + true -> Msg1; + false -> + % First, get the first key and value to set. + Key = hd(keys(Msg2, Opts#{ hashpath => ignore })), + Val = get(Key, Msg2, Opts), + ?event({got_val_to_set, {key, Key}, {val, Val}}), + % Then, set the key and recurse, removing the key from the Msg2. + set(set(Msg1, Key, Val, Opts), remove(Msg2, Key, Opts), Opts) + end. set(Msg1, Key, Value, Opts) -> - % For an individual key, we run deep_set with the key as the path. - % This handles both the case that the key is a path as well as the case - % that it is a single key. - Path = hb_path:term_to_path(Key), - %?event({setting_individual_key, {msg1, Msg1}, {key, Key}, {path, Path}, {value, Value}}), - deep_set(Msg1, Path, Value, Opts). + % For an individual key, we run deep_set with the key as the path. + % This handles both the case that the key is a path as well as the case + % that it is a single key. + Path = hb_path:term_to_path(Key), + %?event({setting_individual_key, {msg1, Msg1}, {key, Key}, {path, Path}, {value, Value}}), + deep_set(Msg1, Path, Value, Opts). %% @doc Recursively search a map, resolving keys, and set the value of the key %% at the given path. deep_set(Msg, [Key], Value, Opts) -> - %?event({setting_last_key, {key, Key}, {value, Value}}), - device_set(Msg, Key, Value, Opts); + %?event({setting_last_key, {key, Key}, {value, Value}}), + device_set(Msg, Key, Value, Opts); deep_set(Msg, [Key|Rest], Value, Opts) -> - {ok, SubMsg} = resolve(Msg, Key, Opts), - ?event({traversing_deeper_to_set, {current_key, Key}, {current_value, SubMsg}, {rest, Rest}}), - device_set(Msg, Key, deep_set(SubMsg, Rest, Value, Opts), Opts). + {ok, SubMsg} = resolve(Msg, Key, Opts), + ?event({traversing_deeper_to_set, {current_key, Key}, {current_value, SubMsg}, {rest, Rest}}), + device_set(Msg, Key, deep_set(SubMsg, Rest, Value, Opts), Opts). device_set(Msg, Key, Value, Opts) -> - ?event({calling_device_set, {key, Key}, {value, Value}}), - ensure_ok(Key, resolve(Msg, #{ path => set, Key => Value }, Opts), Opts). + ?event({calling_device_set, {key, Key}, {value, Value}}), + ensure_ok(Key, resolve(Msg, #{ path => set, Key => Value }, Opts), Opts). %% @doc Remove a key from a message, using its underlying device. remove(Msg, Key) -> remove(Msg, Key, #{}). remove(Msg, Key, Opts) -> - ensure_ok(Key, resolve(Msg, #{ path => remove, item => Key }, Opts), Opts). + ensure_ok(Key, resolve(Msg, #{ path => remove, item => Key }, Opts), Opts). %% @doc Helper for returning `Message2` if the resolution has a status of `ok`, %% or handling an error based on the value of the `error_strategy` option. ensure_ok(_Key, {ok, Value}, _Opts) -> Value; ensure_ok(_Key, {error, Reason}, #{ error_strategy := X }) when X =/= throw -> - {unexpected, {error, Reason}}; + {unexpected, {error, Reason}}; ensure_ok(Key, {error, Reason}, _Opts) -> - throw({pam_resolution_error, Key, Reason}). + throw({pam_resolution_error, Key, Reason}). %% @doc Handle an error in a device call. handle_error(Whence, {Class, Exception, Stacktrace}, Opts) -> - case maps:get(error_strategy, Opts, throw) of - throw -> erlang:raise(Class, Exception, Stacktrace); - _ -> {error, Whence, {Class, Exception, Stacktrace}} - end. + case maps:get(error_strategy, Opts, throw) of + throw -> erlang:raise(Class, Exception, Stacktrace); + _ -> {error, Whence, {Class, Exception, Stacktrace}} + end. %% @doc Truncate the arguments of a function to the number of arguments it %% actually takes. truncate_args(Fun, Args) -> - {arity, Arity} = erlang:fun_info(Fun, arity), - lists:sublist(Args, Arity). + {arity, Arity} = erlang:fun_info(Fun, arity), + lists:sublist(Args, Arity). %% @doc Calculate the Erlang function that should be called to get a value for %% a given key from a device. @@ -283,55 +283,55 @@ truncate_args(Fun, Args) -> %% Returns {ok | add_key, Fun} where Fun is the function to call, and add_key %% indicates that the key should be added to the start of the call's arguments. message_to_fun(Msg, Key) when not is_map_key(device, Msg) -> - % Case 1: The message does not specify a device, so we use the default device. - message_to_fun(Msg#{ device => default_module() }, Key); + % Case 1: The message does not specify a device, so we use the default device. + message_to_fun(Msg#{ device => default_module() }, Key); message_to_fun(Msg = #{ device := RawDev }, Key) -> - ?event({message_to_fun, {msg, Msg}, {device, RawDev}, {key, Key}}), - Dev = - case load_device(RawDev) of - {error, _} -> - % Error case: A device is specified, but it is not loadable. - throw({error, {device_not_loadable, RawDev}}); - {ok, DevMod} -> DevMod - end, - case maps:find(handler, Info = info(Dev, Msg)) of - {ok, Handler} -> - % Case 2: The device has an explicit handler function. - {add_key, Handler}; - error -> - case find_exported_function(Dev, Key, 3) of - {ok, Func} -> - % Case 3: The device has a function of the name `Key`. - {ok, Func}; - not_found -> - case maps:find(default, Info) of - {ok, DefaultFunc} -> - % Case 4: The device has a default handler. - {add_key, DefaultFunc}; - error -> - % Case 5: The device has no default handler. - % We use the default device to handle the key. - case default_module() of - Dev -> - % We are already using the default device, - % so we cannot resolve the key. This should - % never actually happen in practice, but it - % resolves an infinite loop that can occur - % during development. - throw({ - error, - default_device_could_not_resolve_key, - {key, Key} - }); - DefaultDev -> - message_to_fun( - Msg#{ device => DefaultDev }, - Key - ) - end - end - end - end. + ?event({message_to_fun, {msg, Msg}, {device, RawDev}, {key, Key}}), + Dev = + case load_device(RawDev) of + {error, _} -> + % Error case: A device is specified, but it is not loadable. + throw({error, {device_not_loadable, RawDev}}); + {ok, DevMod} -> DevMod + end, + case maps:find(handler, Info = info(Dev, Msg)) of + {ok, Handler} -> + % Case 2: The device has an explicit handler function. + {add_key, Handler}; + error -> + case find_exported_function(Dev, Key, 3) of + {ok, Func} -> + % Case 3: The device has a function of the name `Key`. + {ok, Func}; + not_found -> + case maps:find(default, Info) of + {ok, DefaultFunc} -> + % Case 4: The device has a default handler. + {add_key, DefaultFunc}; + error -> + % Case 5: The device has no default handler. + % We use the default device to handle the key. + case default_module() of + Dev -> + % We are already using the default device, + % so we cannot resolve the key. This should + % never actually happen in practice, but it + % resolves an infinite loop that can occur + % during development. + throw({ + error, + default_device_could_not_resolve_key, + {key, Key} + }); + DefaultDev -> + message_to_fun( + Msg#{ device => DefaultDev }, + Key + ) + end + end + end + end. %% @doc Find the function with the highest arity that has the given name, if it %% exists. @@ -342,65 +342,65 @@ message_to_fun(Msg = #{ device := RawDev }, Key) -> %% the key using its literal value. If that fails, we cast the key to an atom %% and try again. find_exported_function(Dev, Key, MaxArity) when is_map(Dev) -> - case maps:get(Key, Dev, not_found) of - not_found -> - case to_key(Key) of - undefined -> not_found; - Key -> - % The key is unchanged, so we return not_found. - not_found; - KeyAtom -> - % The key was cast to an atom, so we try again. - find_exported_function(Dev, KeyAtom, MaxArity) - end; - Fun when is_function(Fun) -> - case erlang:fun_info(Fun, arity) of - {arity, Arity} when Arity =< MaxArity -> - case is_exported(Dev, Key) of - true -> {ok, Fun}; - false -> not_found - end; - _ -> not_found - end - end; + case maps:get(Key, Dev, not_found) of + not_found -> + case to_key(Key) of + undefined -> not_found; + Key -> + % The key is unchanged, so we return not_found. + not_found; + KeyAtom -> + % The key was cast to an atom, so we try again. + find_exported_function(Dev, KeyAtom, MaxArity) + end; + Fun when is_function(Fun) -> + case erlang:fun_info(Fun, arity) of + {arity, Arity} when Arity =< MaxArity -> + case is_exported(Dev, Key) of + true -> {ok, Fun}; + false -> not_found + end; + _ -> not_found + end + end; find_exported_function(_Mod, _Key, Arity) when Arity < 0 -> not_found; find_exported_function(Mod, Key, Arity) when not is_atom(Key) -> - case to_key(Key) of - ConvertedKey when is_atom(ConvertedKey) -> - find_exported_function(Mod, ConvertedKey, Arity); - undefined -> not_found; - BinaryKey when is_binary(BinaryKey) -> - not_found - end; + case to_key(Key) of + ConvertedKey when is_atom(ConvertedKey) -> + find_exported_function(Mod, ConvertedKey, Arity); + undefined -> not_found; + BinaryKey when is_binary(BinaryKey) -> + not_found + end; find_exported_function(Mod, Key, Arity) -> - case erlang:function_exported(Mod, Key, Arity) of - true -> - case is_exported(Mod, Key) of - true -> {ok, fun Mod:Key/Arity}; - false -> not_found - end; - false -> find_exported_function(Mod, Key, Arity - 1) - end. + case erlang:function_exported(Mod, Key, Arity) of + true -> + case is_exported(Mod, Key) of + true -> {ok, fun Mod:Key/Arity}; + false -> not_found + end; + false -> find_exported_function(Mod, Key, Arity - 1) + end. %% @doc Check if a device is guarding a key via its `exports` list. Defaults to %% true if the device does not specify an `exports` list. The `info` function is %% always exported, if it exists. is_exported(_, info) -> true; is_exported(Dev, Key) -> - case info(Dev) of - #{ exports := Exports } -> - lists:member(Key, Exports); - _ -> true - end. + case info(Dev) of + #{ exports := Exports } -> + lists:member(Key, Exports); + _ -> true + end. %% @doc Convert a key to an atom if it already exists in the Erlang atom table, %% or to a binary otherwise. to_key(Key) -> to_key(Key, #{ error_strategy => throw }). to_key(Key, _Opts) when byte_size(Key) == 43 -> Key; to_key(Key, _Opts) -> - try to_atom_unsafe(Key) - catch _Type:_:_Trace -> key_to_binary(Key) - end. + try to_atom_unsafe(Key) + catch _Type:_:_Trace -> key_to_binary(Key) + end. %% @doc Convert a key to its binary representation. key_to_binary(Key) -> key_to_binary(Key, #{}). @@ -410,12 +410,12 @@ key_to_binary(Key, _Opts) when is_list(Key) -> list_to_binary(Key). %% @doc Helper function for key_to_atom that does not check for errors. to_atom_unsafe(Key) when is_integer(Key) -> - integer_to_binary(Key); + integer_to_binary(Key); to_atom_unsafe(Key) when is_binary(Key) -> - binary_to_existing_atom(hb_util:to_lower(Key), utf8); + binary_to_existing_atom(hb_util:to_lower(Key), utf8); to_atom_unsafe(Key) when is_list(Key) -> - FlattenedKey = lists:flatten(Key), - list_to_existing_atom(FlattenedKey); + FlattenedKey = lists:flatten(Key), + list_to_existing_atom(FlattenedKey); to_atom_unsafe(Key) when is_atom(Key) -> Key. %% @doc Load a device module from its name or a message ID. @@ -428,34 +428,34 @@ load_device(ID, _Opts) when is_atom(ID) -> catch _:_ -> {error, not_loadable} end; load_device(ID, Opts) when is_binary(ID) and byte_size(ID) == 43 -> - case hb_opts:get(load_remote_devices) of - true -> - {ok, Msg} = hb_cache:read_message(maps:get(store, Opts), ID), - Trusted = - lists:any( - fun(Signer) -> - lists:member(Signer, hb_opts:get(trusted_device_signers)) - end, - hb_message:signers(Msg) - ), - case Trusted of - true -> - RelBin = erlang:system_info(otp_release), - case lists:keyfind(<<"Content-Type">>, 1, Msg#tx.tags) of - <<"BEAM/", RelBin/bitstring>> -> - {_, ModNameBin} = - lists:keyfind(<<"Module-Name">>, 1, Msg#tx.tags), - ModName = list_to_atom(binary_to_list(ModNameBin)), - case erlang:load_module(ModName, Msg#tx.data) of - {module, _} -> {ok, ModName}; - {error, Reason} -> {error, Reason} - end - end; - false -> {error, device_signer_not_trusted} - end; - false -> - {error, remote_devices_disabled} - end; + case hb_opts:get(load_remote_devices) of + true -> + {ok, Msg} = hb_cache:read_message(maps:get(store, Opts), ID), + Trusted = + lists:any( + fun(Signer) -> + lists:member(Signer, hb_opts:get(trusted_device_signers)) + end, + hb_message:signers(Msg) + ), + case Trusted of + true -> + RelBin = erlang:system_info(otp_release), + case lists:keyfind(<<"Content-Type">>, 1, Msg#tx.tags) of + <<"BEAM/", RelBin/bitstring>> -> + {_, ModNameBin} = + lists:keyfind(<<"Module-Name">>, 1, Msg#tx.tags), + ModName = list_to_atom(binary_to_list(ModNameBin)), + case erlang:load_module(ModName, Msg#tx.data) of + {module, _} -> {ok, ModName}; + {error, Reason} -> {error, Reason} + end + end; + false -> {error, device_signer_not_trusted} + end; + false -> + {error, remote_devices_disabled} + end; load_device(ID, _Opts) -> case maps:get(ID, hb_opts:get(preloaded_devices), unsupported) of unsupported -> {error, module_not_admissable}; @@ -466,18 +466,18 @@ load_device(ID, _Opts) -> %% device's info function is parameterized by one. info(DevMod) -> info(DevMod, #{}). info(DevMod, Msg) -> - case find_exported_function(DevMod, info, 1) of - {ok, Fun} -> apply(Fun, truncate_args(Fun, [Msg])); - not_found -> #{} - end. + case find_exported_function(DevMod, info, 1) of + {ok, Fun} -> apply(Fun, truncate_args(Fun, [Msg])); + not_found -> #{} + end. %% @doc The default runtime options for a message. At the moment the `Message1` %% but it is included such that we can modulate the options based on the message %% if needed in the future. default_runtime_opts(_Msg1) -> - #{ - error_strategy => throw - }. + #{ + error_strategy => throw + }. %% @doc The default device is the identity device, which simply returns the %% value associated with any key as it exists in its Erlang map. It should also @@ -488,247 +488,247 @@ default_module() -> dev_message. %%% Tests key_from_id_device_test() -> - ?assertEqual({ok, 1}, hb_pam:resolve(#{ a => 1 }, a)). + ?assertEqual({ok, 1}, hb_pam:resolve(#{ a => 1 }, a)). keys_from_id_device_test() -> - ?assertEqual({ok, [a]}, hb_pam:resolve(#{ a => 1, "priv_a" => 2 }, keys)). + ?assertEqual({ok, [a]}, hb_pam:resolve(#{ a => 1, "priv_a" => 2 }, keys)). path_test() -> - ?assertEqual({ok, [test_path]}, hb_pam:resolve(#{ path => [test_path] }, path)), - ?assertEqual({ok, [a]}, hb_pam:resolve(#{ <<"Path">> => [a] }, <<"Path">>)). + ?assertEqual({ok, [test_path]}, hb_pam:resolve(#{ path => [test_path] }, path)), + ?assertEqual({ok, [a]}, hb_pam:resolve(#{ <<"Path">> => [a] }, <<"Path">>)). key_to_binary_test() -> - ?assertEqual(<<"a">>, hb_pam:key_to_binary(a)), - ?assertEqual(<<"a">>, hb_pam:key_to_binary(<<"a">>)), - ?assertEqual(<<"a">>, hb_pam:key_to_binary("a")). + ?assertEqual(<<"a">>, hb_pam:key_to_binary(a)), + ?assertEqual(<<"a">>, hb_pam:key_to_binary(<<"a">>)), + ?assertEqual(<<"a">>, hb_pam:key_to_binary("a")). resolve_binary_key_test() -> - ?assertEqual({ok, 1}, hb_pam:resolve(#{ a => 1 }, <<"a">>)), - ?assertEqual({ok, 1}, hb_pam:resolve(#{ <<"Test-Header">> => 1 }, <<"Test-Header">>)). + ?assertEqual({ok, 1}, hb_pam:resolve(#{ a => 1 }, <<"a">>)), + ?assertEqual({ok, 1}, hb_pam:resolve(#{ <<"Test-Header">> => 1 }, <<"Test-Header">>)). %% @doc Generates a test device with three keys, each of which uses %% progressively more of the arguments that can be passed to a device key. generate_device_with_keys_using_args() -> - #{ - key_using_only_state => - fun(State) -> - {ok, - <<(maps:get(state_key, State))/binary>> - } - end, - key_using_state_and_msg => - fun(State, Msg) -> - {ok, - << - (maps:get(state_key, State))/binary, - (maps:get(msg_key, Msg))/binary - >> - } - end, - key_using_all => - fun(State, Msg, Opts) -> - {ok, - << - (maps:get(state_key, State))/binary, - (maps:get(msg_key, Msg))/binary, - (maps:get(opts_key, Opts))/binary - >> - } - end - }. + #{ + key_using_only_state => + fun(State) -> + {ok, + <<(maps:get(state_key, State))/binary>> + } + end, + key_using_state_and_msg => + fun(State, Msg) -> + {ok, + << + (maps:get(state_key, State))/binary, + (maps:get(msg_key, Msg))/binary + >> + } + end, + key_using_all => + fun(State, Msg, Opts) -> + {ok, + << + (maps:get(state_key, State))/binary, + (maps:get(msg_key, Msg))/binary, + (maps:get(opts_key, Opts))/binary + >> + } + end + }. %% @doc Test that arguments are passed to a device key as expected. %% Particularly, we need to ensure that the key function in the device can %% specify any arity (1 through 3) and the call is handled correctly. key_from_id_device_with_args_test() -> - Msg = - #{ - device => generate_device_with_keys_using_args(), - state_key => <<"1">> - }, - ?assertEqual( - {ok, <<"1">>}, - hb_pam:resolve( - Msg, - #{ - path => key_using_only_state, - msg_key => <<"2">> % Param message, which is ignored - } - ) - ), - ?assertEqual( - {ok, <<"13">>}, - hb_pam:resolve( - Msg, - #{ - path => key_using_state_and_msg, - msg_key => <<"3">> % Param message, with value to add - } - ) - ), - ?assertEqual( - {ok, <<"1337">>}, - hb_pam:resolve( - Msg, - #{ - path => key_using_all, - msg_key => <<"3">> % Param message - }, - #{ - opts_key => <<"37">> % Opts - } - ) - ). + Msg = + #{ + device => generate_device_with_keys_using_args(), + state_key => <<"1">> + }, + ?assertEqual( + {ok, <<"1">>}, + hb_pam:resolve( + Msg, + #{ + path => key_using_only_state, + msg_key => <<"2">> % Param message, which is ignored + } + ) + ), + ?assertEqual( + {ok, <<"13">>}, + hb_pam:resolve( + Msg, + #{ + path => key_using_state_and_msg, + msg_key => <<"3">> % Param message, with value to add + } + ) + ), + ?assertEqual( + {ok, <<"1337">>}, + hb_pam:resolve( + Msg, + #{ + path => key_using_all, + msg_key => <<"3">> % Param message + }, + #{ + opts_key => <<"37">> % Opts + } + ) + ). device_with_handler_function_test() -> - Msg = - #{ - device => - #{ - info => - fun() -> - #{ - handler => - fun(test_key, _S) -> - {ok, <<"GOOD">>} - end - } - end - }, - test_key => <<"BAD">> - }, - ?assertEqual( - {ok, <<"GOOD">>}, - hb_pam:resolve(Msg, test_key) - ). + Msg = + #{ + device => + #{ + info => + fun() -> + #{ + handler => + fun(test_key, _S) -> + {ok, <<"GOOD">>} + end + } + end + }, + test_key => <<"BAD">> + }, + ?assertEqual( + {ok, <<"GOOD">>}, + hb_pam:resolve(Msg, test_key) + ). device_with_default_handler_function_test() -> - Msg = - #{ - device => - #{ - info => - fun() -> - #{ - default => - fun(_, _State) -> - {ok, <<"DEFAULT">>} - end - } - end, - state_key => - fun(_) -> - {ok, <<"STATE">>} - end - } - }, - ?assertEqual( - {ok, <<"STATE">>}, - hb_pam:resolve(Msg, state_key) - ), - ?assertEqual( - {ok, <<"DEFAULT">>}, - hb_pam:resolve(Msg, any_random_key) - ). + Msg = + #{ + device => + #{ + info => + fun() -> + #{ + default => + fun(_, _State) -> + {ok, <<"DEFAULT">>} + end + } + end, + state_key => + fun(_) -> + {ok, <<"STATE">>} + end + } + }, + ?assertEqual( + {ok, <<"STATE">>}, + hb_pam:resolve(Msg, state_key) + ), + ?assertEqual( + {ok, <<"DEFAULT">>}, + hb_pam:resolve(Msg, any_random_key) + ). basic_get_test() -> - Msg = #{ key1 => <<"value1">>, key2 => <<"value2">> }, - ?assertEqual(<<"value1">>, hb_pam:get(key1, Msg)), - ?assertEqual(<<"value2">>, hb_pam:get(key2, Msg)). + Msg = #{ key1 => <<"value1">>, key2 => <<"value2">> }, + ?assertEqual(<<"value1">>, hb_pam:get(key1, Msg)), + ?assertEqual(<<"value2">>, hb_pam:get(key2, Msg)). basic_set_test() -> - Msg = #{ key1 => <<"value1">>, key2 => <<"value2">> }, - UpdatedMsg = hb_pam:set(Msg, #{ key1 => <<"new_value1">> }), - ?event({set_key_complete, {key, key1}, {value, <<"new_value1">>}}), - ?assertEqual(<<"new_value1">>, hb_pam:get(key1, UpdatedMsg)), - ?assertEqual(<<"value2">>, hb_pam:get(key2, UpdatedMsg)). + Msg = #{ key1 => <<"value1">>, key2 => <<"value2">> }, + UpdatedMsg = hb_pam:set(Msg, #{ key1 => <<"new_value1">> }), + ?event({set_key_complete, {key, key1}, {value, <<"new_value1">>}}), + ?assertEqual(<<"new_value1">>, hb_pam:get(key1, UpdatedMsg)), + ?assertEqual(<<"value2">>, hb_pam:get(key2, UpdatedMsg)). get_with_device_test() -> - Msg = - #{ - device => generate_device_with_keys_using_args(), - state_key => <<"STATE">> - }, - ?assertEqual(<<"STATE">>, hb_pam:get(state_key, Msg)), - ?assertEqual(<<"STATE">>, hb_pam:get(key_using_only_state, Msg)). + Msg = + #{ + device => generate_device_with_keys_using_args(), + state_key => <<"STATE">> + }, + ?assertEqual(<<"STATE">>, hb_pam:get(state_key, Msg)), + ?assertEqual(<<"STATE">>, hb_pam:get(key_using_only_state, Msg)). set_with_device_test() -> - Msg = - #{ - device => - #{ - set => - fun(State, _Msg) -> - {ok, - State#{ - set_count => - 1 + maps:get(set_count, State, 0) - } - } - end - }, - state_key => <<"STATE">> - }, - ?assertEqual(<<"STATE">>, hb_pam:get(state_key, Msg)), - SetOnce = hb_pam:set(Msg, #{ state_key => <<"SET_ONCE">> }), - ?assertEqual(1, hb_pam:get(set_count, SetOnce)), - SetTwice = hb_pam:set(SetOnce, #{ state_key => <<"SET_TWICE">> }), - ?assertEqual(2, hb_pam:get(set_count, SetTwice)), - ?assertEqual(<<"STATE">>, hb_pam:get(state_key, SetTwice)). + Msg = + #{ + device => + #{ + set => + fun(State, _Msg) -> + {ok, + State#{ + set_count => + 1 + maps:get(set_count, State, 0) + } + } + end + }, + state_key => <<"STATE">> + }, + ?assertEqual(<<"STATE">>, hb_pam:get(state_key, Msg)), + SetOnce = hb_pam:set(Msg, #{ state_key => <<"SET_ONCE">> }), + ?assertEqual(1, hb_pam:get(set_count, SetOnce)), + SetTwice = hb_pam:set(SetOnce, #{ state_key => <<"SET_TWICE">> }), + ?assertEqual(2, hb_pam:get(set_count, SetTwice)), + ?assertEqual(<<"STATE">>, hb_pam:get(state_key, SetTwice)). deep_set_test() -> - % First validate second layer changes are handled correctly. - Msg0 = #{ a => #{ b => 1 } }, - ?assertMatch(#{ a := #{ b := 2 } }, - hb_pam:set(Msg0, [a, b], 2, #{})), - % Now validate deeper layer changes are handled correctly. - Msg = #{ a => #{ b => #{ c => 1 } } }, - ?assertMatch(#{ a := #{ b := #{ c := 2 } } }, - hb_pam:set(Msg, [a, b, c], 2, #{})). + % First validate second layer changes are handled correctly. + Msg0 = #{ a => #{ b => 1 } }, + ?assertMatch(#{ a := #{ b := 2 } }, + hb_pam:set(Msg0, [a, b], 2, #{})), + % Now validate deeper layer changes are handled correctly. + Msg = #{ a => #{ b => #{ c => 1 } } }, + ?assertMatch(#{ a := #{ b := #{ c := 2 } } }, + hb_pam:set(Msg, [a, b, c], 2, #{})). deep_set_with_device_test() -> - Device = #{ - set => - fun(Msg1, Msg2) -> - % A device where the set function modifies the key - % and adds a modified flag. - {Key, Val} = hd(maps:to_list(maps:remove(path, Msg2))), - {ok, Msg1#{ Key => Val, modified => true }} - end - }, - % A message with an interspersed custom device: A and C have it, - % B does not. A and C will have the modified flag set to true. - Msg = #{ - device => Device, - a => - #{ - b => - #{ - device => Device, - c => 1, - modified => false - }, - modified => false - }, - modified => false - }, - Outer = deep_set(Msg, [a, b, c], 2, #{}), - A = hb_pam:get(a, Outer), - B = hb_pam:get(b, A), - C = hb_pam:get(c, B), - ?assertEqual(2, C), - ?assertEqual(true, hb_pam:get(modified, Outer)), - ?assertEqual(false, hb_pam:get(modified, A)), - ?assertEqual(true, hb_pam:get(modified, B)). + Device = #{ + set => + fun(Msg1, Msg2) -> + % A device where the set function modifies the key + % and adds a modified flag. + {Key, Val} = hd(maps:to_list(maps:remove(path, Msg2))), + {ok, Msg1#{ Key => Val, modified => true }} + end + }, + % A message with an interspersed custom device: A and C have it, + % B does not. A and C will have the modified flag set to true. + Msg = #{ + device => Device, + a => + #{ + b => + #{ + device => Device, + c => 1, + modified => false + }, + modified => false + }, + modified => false + }, + Outer = deep_set(Msg, [a, b, c], 2, #{}), + A = hb_pam:get(a, Outer), + B = hb_pam:get(b, A), + C = hb_pam:get(c, B), + ?assertEqual(2, C), + ?assertEqual(true, hb_pam:get(modified, Outer)), + ?assertEqual(false, hb_pam:get(modified, A)), + ?assertEqual(true, hb_pam:get(modified, B)). device_exports_test() -> - ?assert(is_exported(dev_message, info)), - ?assert(is_exported(dev_message, set)), - ?assert(not is_exported(dev_message, not_exported)), - Dev = #{ - info => fun() -> #{ exports => [set] } end, - set => fun(_, _) -> {ok, <<"SET">>} end - }, - ?assert(is_exported(Dev, info)), - ?assert(is_exported(Dev, set)), - ?assert(not is_exported(Dev, not_exported)). + ?assert(is_exported(dev_message, info)), + ?assert(is_exported(dev_message, set)), + ?assert(not is_exported(dev_message, not_exported)), + Dev = #{ + info => fun() -> #{ exports => [set] } end, + set => fun(_, _) -> {ok, <<"SET">>} end + }, + ?assert(is_exported(Dev, info)), + ?assert(is_exported(Dev, set)), + ?assert(not is_exported(Dev, not_exported)). \ No newline at end of file diff --git a/src/hb_path.erl b/src/hb_path.erl index 5749fa0b..605fafdc 100644 --- a/src/hb_path.erl +++ b/src/hb_path.erl @@ -40,14 +40,14 @@ %% Note: This function uses the `dev_message:get/3` function, rather than %% a generic call as the path should always be an explicit key in the message. hd(Msg2, Opts) -> - %?event({key_from_path, Msg2, Opts}), - case pop_request(Msg2, Opts) of - undefined -> undefined; - {Head, _} -> - % `term_to_path` returns the full path, so we need to take the - % `hd` of our `Head`. - erlang:hd(term_to_path(Head, Opts)) - end. + %?event({key_from_path, Msg2, Opts}), + case pop_request(Msg2, Opts) of + undefined -> undefined; + {Head, _} -> + % `term_to_path` returns the full path, so we need to take the + % `hd` of our `Head`. + erlang:hd(term_to_path(Head, Opts)) + end. %% @doc Return the message without its first path element. Note that this %% is the only transformation in PAM that does _not_ make a log of its @@ -55,103 +55,103 @@ hd(Msg2, Opts) -> %% after executing this transformation. %% This may or may not be the mainnet behavior we want. tl(Msg2, Opts) -> - case pop_request(Msg2, Opts) of - undefined -> undefined; - {_, Rest} when is_map(Rest) -> - maps:get(path, Rest, undefined); - {_, Rest} -> Rest - end. + case pop_request(Msg2, Opts) of + undefined -> undefined; + {_, Rest} when is_map(Rest) -> + maps:get(path, Rest, undefined); + {_, Rest} -> Rest + end. %% @doc Add a path element to a message, according to the type given. push(hashpath, Msg3, Msg2) -> - push_hashpath(Msg3, Msg2); + push_hashpath(Msg3, Msg2); push(request, Msg3, Msg2) -> - push_request(Msg3, Msg2). + push_request(Msg3, Msg2). %%% @doc Add an ID of a Msg2 to the HashPath of another message. push_hashpath(Msg, Msg2) when is_map(Msg2) -> - {ok, Msg2ID} = dev_message:unsigned_id(Msg2), - push_hashpath(Msg, Msg2ID); + {ok, Msg2ID} = dev_message:unsigned_id(Msg2), + push_hashpath(Msg, Msg2ID); push_hashpath(Msg, Msg2ID) -> - ?no_prod("We should use the signed ID if the message is being" - " invoked with it."), - MsgHashpath = from_message(hashpath, Msg), - HashpathFun = hashpath_function(Msg), - NewHashpath = HashpathFun(hb_util:native_id(MsgHashpath), hb_util:native_id(Msg2ID)), - TransformedMsg = Msg#{ hashpath => hb_util:human_id(NewHashpath) }, - % ?event({created_new_hashpath, - % {msg1, hb_util:human_id(MsgHashpath)}, - % {msg2id, hb_util:human_id(Msg2ID)}, - % {new_hashpath, hb_util:human_id(NewHashpath)} - % }), - TransformedMsg. + ?no_prod("We should use the signed ID if the message is being" + " invoked with it."), + MsgHashpath = from_message(hashpath, Msg), + HashpathFun = hashpath_function(Msg), + NewHashpath = HashpathFun(hb_util:native_id(MsgHashpath), hb_util:native_id(Msg2ID)), + TransformedMsg = Msg#{ hashpath => hb_util:human_id(NewHashpath) }, + % ?event({created_new_hashpath, + % {msg1, hb_util:human_id(MsgHashpath)}, + % {msg2id, hb_util:human_id(Msg2ID)}, + % {new_hashpath, hb_util:human_id(NewHashpath)} + % }), + TransformedMsg. %%% @doc Get the hashpath function for a message from its HashPath-Alg. %%% If no hashpath algorithm is specified, the protocol defaults to %%% `sha-256-chain`. hashpath_function(Msg) -> - case dev_message:get(<<"Hashpath-Alg">>, Msg) of - {ok, <<"sha-256-chain">>} -> - fun hb_crypto:sha256_chain/2; - {ok, <<"accumulate-256">>} -> - fun hb_crypto:accumulate/2; - {error, not_found} -> - fun hb_crypto:sha256_chain/2 - end. + case dev_message:get(<<"Hashpath-Alg">>, Msg) of + {ok, <<"sha-256-chain">>} -> + fun hb_crypto:sha256_chain/2; + {ok, <<"accumulate-256">>} -> + fun hb_crypto:accumulate/2; + {error, not_found} -> + fun hb_crypto:sha256_chain/2 + end. %%% @doc Add a message to the head (next to execute) of a request path. push_request(Msg, Path) -> - maps:put(path, term_to_path(Path) ++ from_message(request, Msg), Msg). + maps:put(path, term_to_path(Path) ++ from_message(request, Msg), Msg). %%% @doc Pop the next element from a request path or path list. pop_request(undefined, _Opts) -> undefined; pop_request(Msg, Opts) when is_map(Msg) -> - case pop_request(from_message(request, Msg), Opts) of - undefined -> - ?event(popped_undefined), - undefined; - {undefined, _} -> - undefined; - {Head, Rest} -> - ?event({popped_request, Head, Rest}), - {Head, maps:put(path, Rest, Msg)} - end; + case pop_request(from_message(request, Msg), Opts) of + undefined -> + ?event(popped_undefined), + undefined; + {undefined, _} -> + undefined; + {Head, Rest} -> + ?event({popped_request, Head, Rest}), + {Head, maps:put(path, Rest, Msg)} + end; pop_request([], _Opts) -> undefined; pop_request([Head|Rest], _Opts) -> - {Head, Rest}. + {Head, Rest}. %%% @doc Queue a message at the back of a request path. `path` is the only %%% key that we cannot use dev_message's `set/3` function for (as it expects %%% the compute path to be there), so we use maps:put/3 instead. queue_request(Msg, Path) -> - maps:put(path, from_message(request, Msg) ++ term_to_path(Path), Msg). + maps:put(path, from_message(request, Msg) ++ term_to_path(Path), Msg). %%% @doc Verify the HashPath of a message, given a list of messages that %%% represent its history. Only takes the last message's HashPath-Alg into %%% account, so shouldn't be used in production yet. verify_hashpath(InitialMsg, CurrentMsg, MsgList) when is_map(InitialMsg) -> - {ok, InitialMsgID} = dev_message:unsigned_id(InitialMsg), - verify_hashpath(InitialMsgID, CurrentMsg, MsgList); + {ok, InitialMsgID} = dev_message:unsigned_id(InitialMsg), + verify_hashpath(InitialMsgID, CurrentMsg, MsgList); verify_hashpath(InitialMsgID, CurrentMsg, MsgList) -> - ?no_prod("Must trace if the Hashpath-Alg has changed between messages."), - HashpathFun = hashpath_function(CurrentMsg), - CalculatedHashpath = - lists:foldl( - fun(MsgApplied, Acc) -> - MsgID = - case is_map(MsgApplied) of - true -> - {ok, ID} = dev_message:unsigned_id(MsgApplied), - ID; - false -> MsgApplied - end, - HashpathFun(hb_util:native_id(Acc), hb_util:native_id(MsgID)) - end, - InitialMsgID, - MsgList - ), - CurrentHashpath = from_message(hashpath, CurrentMsg), - hb_util:human_id(CalculatedHashpath) == hb_util:human_id(CurrentHashpath). + ?no_prod("Must trace if the Hashpath-Alg has changed between messages."), + HashpathFun = hashpath_function(CurrentMsg), + CalculatedHashpath = + lists:foldl( + fun(MsgApplied, Acc) -> + MsgID = + case is_map(MsgApplied) of + true -> + {ok, ID} = dev_message:unsigned_id(MsgApplied), + ID; + false -> MsgApplied + end, + HashpathFun(hb_util:native_id(Acc), hb_util:native_id(MsgID)) + end, + InitialMsgID, + MsgList + ), + CurrentHashpath = from_message(hashpath, CurrentMsg), + hb_util:human_id(CalculatedHashpath) == hb_util:human_id(CurrentHashpath). %% @doc Extract the request path or hashpath from a message. We do not use %% PAM for this resolution because this function is called from inside PAM @@ -160,84 +160,84 @@ verify_hashpath(InitialMsgID, CurrentMsg, MsgList) -> %% is directly from a user (in which case paths and hashpaths will not have %% been assigned yet). from_message(hashpath, #{ hashpath := HashPath }) -> - HashPath; + HashPath; from_message(hashpath, Msg) -> - ?no_prod("We should use the signed ID if the message is being" - " invoked with it."), - {ok, Path} = dev_message:unsigned_id(Msg), - hd(term_to_path(Path)); + ?no_prod("We should use the signed ID if the message is being" + " invoked with it."), + {ok, Path} = dev_message:unsigned_id(Msg), + hd(term_to_path(Path)); from_message(request, #{ path := [] }) -> undefined; from_message(request, #{ path := Path }) when is_list(Path) -> - term_to_path(Path); + term_to_path(Path); from_message(request, #{ path := Other }) -> - term_to_path(Other). + term_to_path(Other). %% @doc Convert a term into an executable path. Supports binaries, lists, and %% atoms. Notably, it does not support strings as lists of characters. term_to_path(Path) -> term_to_path(Path, #{ error_strategy => throw }). term_to_path(Binary, Opts) when is_binary(Binary) -> - case binary:match(Binary, <<"/">>) of - nomatch -> [Binary]; - _ -> - term_to_path( - lists:filter( - fun(Part) -> byte_size(Part) > 0 end, - binary:split(Binary, <<"/">>, [global]) - ), - Opts - ) - end; + case binary:match(Binary, <<"/">>) of + nomatch -> [Binary]; + _ -> + term_to_path( + lists:filter( + fun(Part) -> byte_size(Part) > 0 end, + binary:split(Binary, <<"/">>, [global]) + ), + Opts + ) + end; term_to_path([], _Opts) -> undefined; term_to_path(List, Opts) when is_list(List) -> - lists:map(fun(Part) -> hb_pam:to_key(Part, Opts) end, List); + lists:map(fun(Part) -> hb_pam:to_key(Part, Opts) end, List); term_to_path(Atom, _Opts) when is_atom(Atom) -> [Atom]. %%% TESTS push_hashpath_test() -> - Msg1 = #{ <<"empty">> => <<"message">> }, - Msg2 = #{ <<"exciting">> => <<"message2">> }, - Msg3 = push_hashpath(Msg1, Msg2), - ?assert(is_binary(maps:get(hashpath, Msg3))). + Msg1 = #{ <<"empty">> => <<"message">> }, + Msg2 = #{ <<"exciting">> => <<"message2">> }, + Msg3 = push_hashpath(Msg1, Msg2), + ?assert(is_binary(maps:get(hashpath, Msg3))). push_multiple_hashpaths_test() -> - Msg1 = #{ <<"empty">> => <<"message">> }, - Msg2 = #{ <<"exciting">> => <<"message2">> }, - Msg3 = push_hashpath(Msg1, Msg2), - Msg4 = #{ <<"exciting">> => <<"message4">> }, - Msg5 = push_hashpath(Msg3, Msg4), - ?assert(is_binary(maps:get(hashpath, Msg5))). + Msg1 = #{ <<"empty">> => <<"message">> }, + Msg2 = #{ <<"exciting">> => <<"message2">> }, + Msg3 = push_hashpath(Msg1, Msg2), + Msg4 = #{ <<"exciting">> => <<"message4">> }, + Msg5 = push_hashpath(Msg3, Msg4), + ?assert(is_binary(maps:get(hashpath, Msg5))). verify_hashpath_test() -> - Msg1 = #{ <<"empty">> => <<"message">> }, - MsgB1 = #{ <<"exciting1">> => <<"message1">> }, - MsgB2 = #{ <<"exciting2">> => <<"message2">> }, - MsgB3 = #{ <<"exciting3">> => <<"message3">> }, - MsgR1 = push_hashpath(Msg1, MsgB1), - MsgR2 = push_hashpath(MsgR1, MsgB2), - MsgR3 = push_hashpath(MsgR2, MsgB3), - ?assert(verify_hashpath(Msg1, MsgR3, [MsgB1, MsgB2, MsgB3])). + Msg1 = #{ <<"empty">> => <<"message">> }, + MsgB1 = #{ <<"exciting1">> => <<"message1">> }, + MsgB2 = #{ <<"exciting2">> => <<"message2">> }, + MsgB3 = #{ <<"exciting3">> => <<"message3">> }, + MsgR1 = push_hashpath(Msg1, MsgB1), + MsgR2 = push_hashpath(MsgR1, MsgB2), + MsgR3 = push_hashpath(MsgR2, MsgB3), + ?assert(verify_hashpath(Msg1, MsgR3, [MsgB1, MsgB2, MsgB3])). validate_path_transitions(X, Opts) -> - {Head, X2} = pop_request(X, Opts), - ?assertEqual(a, Head), - {H2, X3} = pop_request(X2, Opts), - ?assertEqual(b, H2), - {H3, X4} = pop_request(X3, Opts), - ?assertEqual(c, H3), - ?assertEqual(undefined, pop_request(X4, Opts)). + {Head, X2} = pop_request(X, Opts), + ?assertEqual(a, Head), + {H2, X3} = pop_request(X2, Opts), + ?assertEqual(b, H2), + {H3, X4} = pop_request(X3, Opts), + ?assertEqual(c, H3), + ?assertEqual(undefined, pop_request(X4, Opts)). pop_from_message_test() -> - validate_path_transitions(#{ path => [a, b, c] }, #{}). + validate_path_transitions(#{ path => [a, b, c] }, #{}). pop_from_path_list_test() -> - validate_path_transitions([a, b, c], #{}). + validate_path_transitions([a, b, c], #{}). hd_test() -> - ?assertEqual(a, hd(#{ path => [a, b, c] }, #{})), - ?assertEqual(undefined, hd(#{ path => undefined }, #{})). + ?assertEqual(a, hd(#{ path => [a, b, c] }, #{})), + ?assertEqual(undefined, hd(#{ path => undefined }, #{})). tl_test() -> - ?assertMatch([b, c], tl(#{ path => [a, b, c] }, #{})), - ?assertEqual(undefined, tl(#{ path => [] }, #{})), - ?assertEqual(undefined, tl(#{ path => undefined }, #{})). \ No newline at end of file + ?assertMatch([b, c], tl(#{ path => [a, b, c] }, #{})), + ?assertEqual(undefined, tl(#{ path => [] }, #{})), + ?assertEqual(undefined, tl(#{ path => undefined }, #{})). \ No newline at end of file diff --git a/src/hb_private.erl b/src/hb_private.erl index adc760ed..7244e054 100644 --- a/src/hb_private.erl +++ b/src/hb_private.erl @@ -20,51 +20,51 @@ from_message(Msg) -> maps:get(private, Msg, #{}). %% @doc Helper for getting a value from the private element of a message. get(Msg, Key) -> - get(Msg, Key, undefined). + get(Msg, Key, undefined). get(Msg, Key, Default) -> - maps:get(Key, from_message(Msg), Default). + maps:get(Key, from_message(Msg), Default). %% @doc Helper function for setting a key in the private element of a message. set(Msg, Key, Value) -> - maps:put( - private, - maps:put(Key, Value, from_message(Msg)), - Msg - ). + maps:put( + private, + maps:put(Key, Value, from_message(Msg)), + Msg + ). %% @doc Check if a key is private. is_private(Key) -> - Str = key_to_list(Key), - lists:prefix("priv", Str). + Str = key_to_list(Key), + lists:prefix("priv", Str). %% @doc Convert a key to a list. key_to_list(Key) when is_atom(Key) -> - atom_to_list(Key); + atom_to_list(Key); key_to_list(Key) when is_binary(Key) -> - binary_to_list(Key); + binary_to_list(Key); key_to_list(Key) when is_list(Key) -> - binary_to_list(iolist_to_binary(Key)). + binary_to_list(iolist_to_binary(Key)). %% @doc Unset all of the private keys in a message. reset(Msg) -> - maps:without( - lists:filter(fun is_private/1, maps:keys(Msg)), - Msg - ). + maps:without( + lists:filter(fun is_private/1, maps:keys(Msg)), + Msg + ). %%% Tests set_private_test() -> - ?assertEqual(#{a => 1, private => #{b => 2}}, ?MODULE:set(#{a => 1}, b, 2)), - Res = ?MODULE:set(#{a => 1}, a, 1), - ?assertEqual(#{a => 1, private => #{a => 1}}, Res), - ?assertEqual(#{a => 1, private => #{a => 1}}, ?MODULE:set(Res, a, 1)). + ?assertEqual(#{a => 1, private => #{b => 2}}, ?MODULE:set(#{a => 1}, b, 2)), + Res = ?MODULE:set(#{a => 1}, a, 1), + ?assertEqual(#{a => 1, private => #{a => 1}}, Res), + ?assertEqual(#{a => 1, private => #{a => 1}}, ?MODULE:set(Res, a, 1)). get_private_key_test() -> - M1 = #{a => 1, private => #{b => 2}}, - ?assertEqual(undefined, ?MODULE:get(M1, a)), - {ok, [a]} = hb_pam:resolve(M1, <<"Keys">>), - ?assertEqual(2, ?MODULE:get(M1, b)), - {Res, _} = hb_pam:resolve(M1, <<"Private">>), - ?assertNotEqual(ok, Res), - {Res, _} = hb_pam:resolve(M1, private). \ No newline at end of file + M1 = #{a => 1, private => #{b => 2}}, + ?assertEqual(undefined, ?MODULE:get(M1, a)), + {ok, [a]} = hb_pam:resolve(M1, <<"Keys">>), + ?assertEqual(2, ?MODULE:get(M1, b)), + {Res, _} = hb_pam:resolve(M1, <<"Private">>), + ?assertNotEqual(ok, Res), + {Res, _} = hb_pam:resolve(M1, private). \ No newline at end of file diff --git a/src/hb_process.erl b/src/hb_process.erl index 8eaf7fa2..14ee3f60 100644 --- a/src/hb_process.erl +++ b/src/hb_process.erl @@ -96,12 +96,12 @@ result(RawProcID, RawMsgRef, Store, Wallet) -> ?event({found_cu_for_proc, hb_util:id(ProcID)}), ?no_prod("The CU process IPC API is poorly named."), Pid ! - { - on_idle, - run, - add_monitor, - [create_monitor_for_message(MsgRef)] - }, + { + on_idle, + run, + add_monitor, + [create_monitor_for_message(MsgRef)] + }, Pid ! {on_idle, message, MsgRef}, ?event({added_listener_and_message, Pid, MsgRef}), await_results(Pid) @@ -137,45 +137,45 @@ create_monitor_for_message(Msg) when is_record(Msg, tx) -> create_monitor_for_message(MsgID) -> Listener = self(), fun(S, {message, Inbound}) -> - % Gather messages + % Gather messages Assignment = maps:get(<<"Assignment">>, Inbound#tx.data), - Msg = maps:get(<<"Message">>, Inbound#tx.data), - % Gather IDs + Msg = maps:get(<<"Message">>, Inbound#tx.data), + % Gather IDs AssignmentID = hb_util:id(Assignment, signed), AssignmentUnsignedID = hb_util:id(Assignment, unsigned), ScheduledMsgID = hb_util:id(Msg, signed), ScheduledMsgUnsignedID = hb_util:id(Msg, unsigned), - % Gather slot + % Gather slot Slot = case lists:keyfind(<<"Slot">>, 1, Assignment#tx.tags) of {<<"Slot">>, RawSlot} -> list_to_integer(binary_to_list(RawSlot)); false -> no_slot end, - % Check if the message is relevant + % Check if the message is relevant IsRelevant = (Slot == MsgID) or (ScheduledMsgID == MsgID) or (AssignmentID == MsgID) or (ScheduledMsgUnsignedID == MsgID) or (AssignmentUnsignedID == MsgID), - % Send the result if the message is relevant. - % Continue waiting otherwise. + % Send the result if the message is relevant. + % Continue waiting otherwise. case IsRelevant of true -> Listener ! {result, self(), Inbound, S}, done; false -> ?event({monitor_got_message_for_wrong_slot, Slot, MsgID}), - %ar_bundles:print(Inbound), + %ar_bundles:print(Inbound), ignored end; (S, end_of_schedule) -> ?event( - {monitor_got_eos, - {waiting_for_slot, maps:get(slot, S, no_slot)}, - {to, maps:get(to, S, no_to)} - } - ), + {monitor_got_eos, + {waiting_for_slot, maps:get(slot, S, no_slot)}, + {to, maps:get(to, S, no_to)} + } + ), ignored; (_, Signal) -> ?event({monitor_got_unknown_signal, Signal}), @@ -206,11 +206,11 @@ create_persistent_monitor() -> boot(Process, Opts) -> % Register the process so that it can be found by its ID. ?event( - {booting_process, - {signed, hb_util:id(Process, signed)}, - {unsigned, hb_util:id(Process, unsigned)} - } - ), + {booting_process, + {signed, hb_util:id(Process, signed)}, + {unsigned, hb_util:id(Process, unsigned)} + } + ), pg:join({cu, hb_util:id(Process, signed)}, self()), % Build the device stack. ?event({registered_process, hb_util:id(Process, signed)}), @@ -227,7 +227,7 @@ boot(Process, Opts) -> {ok, #{keys := [Key|_]}} = hb_pam:resolve(Dev, checkpoint_uses, [BootState], Opts), % We don't support partial checkpoints (perhaps we never will?), so just - % take one key and use that to find the latest full checkpoint. + % take one key and use that to find the latest full checkpoint. CheckpointOption = hb_cache:latest( Store, @@ -255,19 +255,19 @@ boot(Process, Opts) -> devices => Devs }, ?event( - {running_init_on_slot, - Slot + 1, - maps:get(to, Opts, inf), - maps:keys(Checkpoint) - } - ), + {running_init_on_slot, + Slot + 1, + maps:get(to, Opts, inf), + maps:keys(Checkpoint) + } + ), RuntimeOpts = - Opts#{ - proc_dev => Dev, - return => all, - % If no compute mode is already set, use the global default. - compute_mode => maps:get(compute_mode, Opts, hb_opts:get(compute_mode)) - }, + Opts#{ + proc_dev => Dev, + return => all, + % If no compute mode is already set, use the global default. + compute_mode => maps:get(compute_mode, Opts, hb_opts:get(compute_mode)) + }, case hb_pam:resolve(Dev, init, [InitState, RuntimeOpts]) of {ok, StateAfterInit} -> execute_schedule(StateAfterInit, RuntimeOpts); @@ -276,44 +276,44 @@ boot(Process, Opts) -> end. execute_schedule(State, Opts) -> - ?event( - { - process_executing_slot, - maps:get(slot, State), - {proc_id, hb_util:id(maps:get(process, State))}, - {to, maps:get(to, State)} - } - ), + ?event( + { + process_executing_slot, + maps:get(slot, State), + {proc_id, hb_util:id(maps:get(process, State))}, + {to, maps:get(to, State)} + } + ), case State of #{schedule := []} -> - ?event( - {process_finished_schedule, - {final_slot, maps:get(slot, State)} - } - ), - case maps:get(compute_mode, Opts) of - aggressive -> - case execute_eos(State, Opts) of - {ok, #{schedule := []}} -> - ?event(eos_did_not_yield_more_work), - await_command(State, Opts); - {ok, NS} -> - execute_schedule(NS, Opts); - {error, DevNum, DevMod, Info} -> - ?event({error, {DevNum, DevMod, Info}}), - execute_terminate( - State#{ - errors := - maps:get(errors, State, []) - ++ [{DevNum, DevMod, Info}] - }, - Opts - ) - end; - lazy -> - ?event({lazy_compute_mode, moving_to_await_state}), - await_command(State, Opts) - end; + ?event( + {process_finished_schedule, + {final_slot, maps:get(slot, State)} + } + ), + case maps:get(compute_mode, Opts) of + aggressive -> + case execute_eos(State, Opts) of + {ok, #{schedule := []}} -> + ?event(eos_did_not_yield_more_work), + await_command(State, Opts); + {ok, NS} -> + execute_schedule(NS, Opts); + {error, DevNum, DevMod, Info} -> + ?event({error, {DevNum, DevMod, Info}}), + execute_terminate( + State#{ + errors := + maps:get(errors, State, []) + ++ [{DevNum, DevMod, Info}] + }, + Opts + ) + end; + lazy -> + ?event({lazy_compute_mode, moving_to_await_state}), + await_command(State, Opts) + end; #{schedule := [Msg | NextSched]} -> case execute_message(Msg, State, Opts) of {ok, NewState = #{schedule := [Msg | NextSched]}} -> @@ -379,7 +379,7 @@ post_execute( case is_record(Item, tx) of true -> {Key, Item}; false -> - throw({error, checkpoint_result_not_tx, Key}) + throw({error, checkpoint_result_not_tx, Key}) end end, maps:get(save_keys, CheckpointState, []) @@ -397,7 +397,7 @@ post_execute( ?event({checkpoint_written_for_slot, Slot}); false -> NormalizedResult = - ar_bundles:deserialize(ar_bundles:serialize(Results)), + ar_bundles:deserialize(ar_bundles:serialize(Results)), hb_cache:write_output( Store, hb_util:id(Process, signed), @@ -419,27 +419,27 @@ initialize_slot(State = #{slot := Slot}) -> execute_message(Msg, State, Opts = #{ proc_dev := Dev }) -> hb_pam:resolve( - Dev, - execute, - [State#{ message => Msg }, Opts], - Opts - ). + Dev, + execute, + [State#{ message => Msg }, Opts], + Opts + ). execute_terminate(S, Opts = #{ proc_dev := Dev }) -> hb_pam:resolve( - Dev, - terminate, - [S#{ message => undefined }, Opts], - Opts - ). + Dev, + terminate, + [S#{ message => undefined }, Opts], + Opts + ). execute_eos(S, Opts = #{ proc_dev := Dev }) -> hb_pam:resolve( - Dev, - end_of_schedule, - [S#{ message => undefined }, Opts], - Opts - ). + Dev, + end_of_schedule, + [S#{ message => undefined }, Opts], + Opts + ). is_checkpoint_slot(State, Opts) -> (maps:get(is_checkpoint, Opts, fun(_) -> false end))(State) @@ -455,11 +455,11 @@ await_command(State, Opts = #{ on_idle := wait, proc_dev := Dev }) -> {on_idle, run, Function, Args} -> ?event({running_command, Function, Args}), {ok, NewState} = hb_pam:resolve( - Dev, - Function, - [State#{ message => hd(Args) }, Opts], - Opts - ), + Dev, + Function, + [State#{ message => hd(Args) }, Opts], + Opts + ), await_command(NewState, Opts); {on_idle, message, MsgRef} -> ?event({received_message, MsgRef}), @@ -473,4 +473,4 @@ await_command(State, Opts = #{ on_idle := wait, proc_dev := Dev }) -> Other -> ?event({received_unknown_message, Other}), await_command(State, Opts) - end. + end. \ No newline at end of file diff --git a/src/hb_process_monitor.erl b/src/hb_process_monitor.erl index d1118f3f..9aa8647c 100644 --- a/src/hb_process_monitor.erl +++ b/src/hb_process_monitor.erl @@ -42,9 +42,9 @@ handle_crons(State) -> {ok, HasNextPage, Results, Cursor} -> lists:map( fun(Res) -> - % TODO: Validate this - dev_mu:push(#{ message => Res }, State) - end, + % TODO: Validate this + dev_mu:push(#{ message => Res }, State) + end, Results ), NS = State#state{cursor = Cursor}, diff --git a/src/hb_router.erl b/src/hb_router.erl index b5304ce6..c48558cd 100644 --- a/src/hb_router.erl +++ b/src/hb_router.erl @@ -13,4 +13,4 @@ find(Type, _ID, Address) -> case maps:get(Type, hb_opts:get(nodes), undefined) of #{ Address := Node } -> {ok, Node}; undefined -> {error, service_type_not_found} - end. + end. \ No newline at end of file diff --git a/src/hb_store.erl b/src/hb_store.erl index 133d18b2..fda0aea0 100644 --- a/src/hb_store.erl +++ b/src/hb_store.erl @@ -51,17 +51,17 @@ scope(Store, Scope) -> Store, fun(StoreScope, _) -> StoreScope == Scope orelse - (is_list(Scope) andalso lists:member(StoreScope, Scope)) + (is_list(Scope) andalso lists:member(StoreScope, Scope)) end ). %% @doc Ask a store for its own scope. If it doesn't have one, return the %% default scope (local). get_store_scope(Store) -> - case call_function(Store, scope, []) of - not_found -> ?DEFAULT_SCOPE; - Scope -> Scope - end. + case call_function(Store, scope, []) of + not_found -> ?DEFAULT_SCOPE; + Scope -> Scope + end. %% @doc Order a store by a preference of its scopes. This is useful for making %% sure that faster (or perhaps cheaper) stores are used first. If a list is @@ -69,28 +69,28 @@ get_store_scope(Store) -> %% scopes will be ordered by the scores in the map. Any unknown scopes will %% default to a score of 0. sort(Stores, PreferenceOrder) when is_list(PreferenceOrder) -> - sort( - Stores, - maps:from_list( - [ - {Scope, -Index} - || - {Scope, Index} <- - lists:zip( - PreferenceOrder, - lists:seq(1, length(PreferenceOrder)) - ) - ] - ) - ); + sort( + Stores, + maps:from_list( + [ + {Scope, -Index} + || + {Scope, Index} <- + lists:zip( + PreferenceOrder, + lists:seq(1, length(PreferenceOrder)) + ) + ] + ) + ); sort(Stores, ScoreMap) -> - lists:sort( - fun(Store1, Store2) -> - maps:get(get_store_scope(Store1), ScoreMap, 0) > - maps:get(get_store_scope(Store2), ScoreMap, 0) - end, - Stores - ). + lists:sort( + fun(Store1, Store2) -> + maps:get(get_store_scope(Store1), ScoreMap, 0) > + maps:get(get_store_scope(Store2), ScoreMap, 0) + end, + Stores + ). %% @doc Join a list of path components together. join([]) -> []; @@ -98,7 +98,7 @@ join(Path) when is_binary(Path) -> Path; join([""|Xs]) -> join(Xs); join(FN = [X|_Xs]) when is_integer(X) -> FN; join([X|Xs]) -> - filename:join(join(X), join(Xs)). + filename:join(join(X), join(Xs)). %%% The store interface that modules should implement. @@ -114,7 +114,7 @@ make_group(Modules, Path) -> call_function(Modules, make_group, [Path]). %% @doc Make a link from one path to another in the store. make_link(Modules, Existing, New) -> - call_function(Modules, make_link, [Existing, New]). + call_function(Modules, make_link, [Existing, New]). %% @doc Delete all of the keys in a store. Should be used with extreme %% caution. Lost data can lose money in many/most of hyperbeam's use cases. diff --git a/src/hb_store_fs.erl b/src/hb_store_fs.erl index 563e9663..4b1f4c62 100644 --- a/src/hb_store_fs.erl +++ b/src/hb_store_fs.erl @@ -111,8 +111,8 @@ make_link(Opts, Existing, New) -> %% @doc Add the directory prefix to a path. add_prefix(#{ prefix := Prefix }, Path) -> - hb_store:join([Prefix, Path]). + hb_store:join([Prefix, Path]). %% @doc Remove the directory prefix from a path. remove_prefix(#{ prefix := Prefix }, Path) -> - hb_util:remove_common(Path, Prefix). \ No newline at end of file + hb_util:remove_common(Path, Prefix). \ No newline at end of file diff --git a/src/hb_store_remote_node.erl b/src/hb_store_remote_node.erl index dc355cfb..4ed66594 100644 --- a/src/hb_store_remote_node.erl +++ b/src/hb_store_remote_node.erl @@ -37,4 +37,4 @@ read(Opts = #{ node := Node }, Key) -> {ok, Bundle} end; Error -> Error - end. + end. \ No newline at end of file diff --git a/src/hb_sup.erl b/src/hb_sup.erl index 2ce3472b..613f0eef 100644 --- a/src/hb_sup.erl +++ b/src/hb_sup.erl @@ -27,9 +27,9 @@ start_link() -> %% modules => modules()} % optional init([]) -> SupFlags = #{strategy => one_for_all, - intensity => 0, - period => 1}, + intensity => 0, + period => 1}, ChildSpecs = [], {ok, {SupFlags, ChildSpecs}}. -%% internal functions +%% internal functions \ No newline at end of file diff --git a/src/hb_test.erl b/src/hb_test.erl index 6fa0c1ee..6b8a5c95 100644 --- a/src/hb_test.erl +++ b/src/hb_test.erl @@ -153,4 +153,4 @@ generate_test_data(Script, Wallet, _Opts, Devs) -> ), hb_cache:write(Store, Msg), ?event({test_data_written, {proc, hb_util:id(SignedProcess, signed)}, {msg, hb_util:id(Msg, unsigned)}}), - {SignedProcess, Msg}. + {SignedProcess, Msg}. \ No newline at end of file diff --git a/src/hb_util.erl b/src/hb_util.erl index ed4c3972..871be62c 100644 --- a/src/hb_util.erl +++ b/src/hb_util.erl @@ -18,38 +18,38 @@ %% a message explicitly, raw encoded ID, or an Erlang Arweave `tx` record. id(Item) -> id(Item, unsigned). id(TX, Type) when is_record(TX, tx) -> - encode(ar_bundles:id(TX, Type)); + encode(ar_bundles:id(TX, Type)); id(Map, Type) when is_map(Map) -> - case Type of - unsigned -> hb_pam:get(unsigned_id, Map); - signed -> encode(hb_pam:get(id, Map)) - end; + case Type of + unsigned -> hb_pam:get(unsigned_id, Map); + signed -> encode(hb_pam:get(id, Map)) + end; id(Bin, _) when is_binary(Bin) andalso byte_size(Bin) == 43 -> - Bin; + Bin; id(Bin, _) when is_binary(Bin) andalso byte_size(Bin) == 32 -> - encode(Bin); + encode(Bin); id(Data, Type) when is_list(Data) -> - id(list_to_binary(Data), Type). + id(list_to_binary(Data), Type). %% @doc Convert a string to a lowercase. to_lower(Str) when is_list(Str) -> - string:to_lower(Str); + string:to_lower(Str); to_lower(Bin) when is_binary(Bin) -> - list_to_binary(to_lower(binary_to_list(Bin))). + list_to_binary(to_lower(binary_to_list(Bin))). %% @doc Convert a human readable ID to a native binary ID. If the ID is already %% a native binary ID, it is returned as is. native_id(Bin) when is_binary(Bin) andalso byte_size(Bin) == 43 -> - decode(Bin); + decode(Bin); native_id(Bin) when is_binary(Bin) andalso byte_size(Bin) == 32 -> - Bin. + Bin. %% @doc Convert a native binary ID to a human readable ID. If the ID is already %% a human readable ID, it is returned as is. human_id(Bin) when is_binary(Bin) andalso byte_size(Bin) == 32 -> - encode(Bin); + encode(Bin); human_id(Bin) when is_binary(Bin) andalso byte_size(Bin) == 43 -> - Bin. + Bin. %% @doc Encode a binary to URL safe base64 binary string. encode(Bin) -> @@ -74,15 +74,15 @@ safe_decode(E) -> {ok, D} catch _:_ -> - {error, invalid} + {error, invalid} end. %% @doc Label a list of elements with a number. number(List) -> - lists:map( - fun({N, Item}) -> {integer_to_binary(N), Item} end, - lists:zip(lists:seq(1, length(List)), List) - ). + lists:map( + fun({N, Item}) -> {integer_to_binary(N), Item} end, + lists:zip(lists:seq(1, length(List)), List) + ). %% @doc Convert a list of elements to a map with numbered keys. list_to_numbered_map(List) -> @@ -92,24 +92,24 @@ list_to_numbered_map(List) -> %% with the associated key as an integer and a value. Optionally, it takes a %% standard map of HyperBEAM runtime options. message_to_numbered_list(Message) -> - message_to_numbered_list(Message, #{}). + message_to_numbered_list(Message, #{}). message_to_numbered_list(Message, Opts) -> - {ok, Keys} = hb_pam:keys(Message, Opts), - KeyValList = - lists:filtermap( - fun(Key) -> - case string:to_integer(Key) of - {Int, ""} -> - { - true, - {Int, hb_pam:get(Key, Message, Opts)} - }; - _ -> false - end - end, - Keys - ), - lists:sort(KeyValList). + {ok, Keys} = hb_pam:keys(Message, Opts), + KeyValList = + lists:filtermap( + fun(Key) -> + case string:to_integer(Key) of + {Int, ""} -> + { + true, + {Int, hb_pam:get(Key, Message, Opts)} + }; + _ -> false + end + end, + Keys + ), + lists:sort(KeyValList). %% @doc Convert a map of numbered elements to a list. We stop at the first %% integer key that is not associated with a value. @@ -125,23 +125,23 @@ message_to_numbered_list(Message, Opts) -> %% message will not lead to an exception. hd(Message) -> hd(Message, value). hd(Message, ReturnType) -> - hd(Message, ReturnType, #{ error_strategy => throw }). + hd(Message, ReturnType, #{ error_strategy => throw }). hd(Message, ReturnType, Opts) -> - {ok, Keys} = hb_pam:resolve(Message, keys), - hd(Message, Keys, 1, ReturnType, Opts). + {ok, Keys} = hb_pam:resolve(Message, keys), + hd(Message, Keys, 1, ReturnType, Opts). hd(_Map, [], _Index, _ReturnType, #{ error_strategy := throw }) -> - throw(no_integer_keys); + throw(no_integer_keys); hd(_Map, [], _Index, _ReturnType, _Opts) -> undefined; hd(Message, [Key|Rest], Index, ReturnType, Opts) -> - case hb_pam:to_key(Key, Opts#{ error_strategy => return }) of - undefined -> - hd(Message, Rest, Index + 1, ReturnType, Opts); - Key -> - case ReturnType of - key -> Key; - value -> hb_pam:resolve(Message, Key) - end - end. + case hb_pam:to_key(Key, Opts#{ error_strategy => return }) of + undefined -> + hd(Message, Rest, Index + 1, ReturnType, Opts); + Key -> + case ReturnType of + key -> Key; + value -> hb_pam:resolve(Message, Key) + end + end. %% @doc Find the value associated with a key in parsed a JSON structure list. find_value(Key, List) -> @@ -150,16 +150,16 @@ find_value(Key, List) -> find_value(Key, Map, Default) when is_map(Map) -> case maps:find(Key, Map) of {ok, Value} -> - Value; + Value; error -> - Default + Default end; find_value(Key, List, Default) -> case lists:keyfind(Key, 1, List) of {Key, Val} -> - Val; + Val; false -> - Default + Default end. %% @doc Remove the common prefix from two strings, returning the remainder of the @@ -179,10 +179,10 @@ remove_common(Rest, _) -> Rest. %% @doc Throw an exception if the Opts map has an `error_strategy` key with the %% value `throw`. Otherwise, return the value. maybe_throw(Val, Opts) -> - case hb_pam:get(error_strategy, Opts) of - throw -> throw(Val); - _ -> Val - end. + case hb_pam:get(error_strategy, Opts) of + throw -> throw(Val); + _ -> Val + end. %% @doc Print a message to the standard error stream, prefixed by the amount %% of time that has elapsed since the last call to this function. @@ -192,9 +192,9 @@ debug_print(X, Mod, Func, LineNum) -> TSDiff = case Last of undefined -> 0; _ -> Now - Last end, io:format(standard_error, "=== HB DEBUG ===[~pms in ~p @ ~s:~w ~p]==> ~s~n", [ - TSDiff, self(), Mod, LineNum, Func, - lists:flatten(debug_fmt(X, 0)) - ]), + TSDiff, self(), Mod, LineNum, Func, + lists:flatten(debug_fmt(X, 0)) + ]), X. %% @doc Convert a term to a string for debugging print purposes. @@ -205,26 +205,26 @@ debug_fmt({X, Y}, Indent) when is_atom(X) and is_atom(Y) -> format_indented("~p: ~p", [X, Y], Indent); debug_fmt({X, Y}, Indent) when is_record(Y, tx) -> format_indented("~p: [TX item]~n~s", - [X, ar_bundles:format(Y, Indent + 1)], - Indent - ); + [X, ar_bundles:format(Y, Indent + 1)], + Indent + ); debug_fmt({X, Y}, Indent) when is_map(Y) -> - Formatted = hb_util:format_map(Y, Indent + 1), - HasNewline = lists:member($\n, Formatted), - format_indented("~p~s", - [ - X, - case HasNewline of - true -> " ==>" ++ Formatted; - false -> ": " ++ Formatted - end - ], - Indent - ); + Formatted = hb_util:format_map(Y, Indent + 1), + HasNewline = lists:member($\n, Formatted), + format_indented("~p~s", + [ + X, + case HasNewline of + true -> " ==>" ++ Formatted; + false -> ": " ++ Formatted + end + ], + Indent + ); debug_fmt({X, Y}, Indent) -> format_indented("~s: ~s", [debug_fmt(X, Indent), debug_fmt(Y, Indent)], Indent); debug_fmt(Map, Indent) when is_map(Map) -> - hb_util:format_map(Map, Indent); + hb_util:format_map(Map, Indent); debug_fmt(Tuple, Indent) when is_tuple(Tuple) -> format_tuple(Tuple, Indent); debug_fmt(Str = [X | _], Indent) when is_integer(X) andalso X >= 32 andalso X < 127 -> @@ -234,73 +234,73 @@ debug_fmt(X, Indent) -> %% @doc Helper function to format tuples with arity greater than 2. format_tuple(Tuple, Indent) -> - to_lines(lists:map( - fun(Elem) -> - debug_fmt(Elem, Indent) - end, - tuple_to_list(Tuple) - )). + to_lines(lists:map( + fun(Elem) -> + debug_fmt(Elem, Indent) + end, + tuple_to_list(Tuple) + )). to_lines([]) -> []; to_lines(In =[RawElem | Rest]) -> - Elem = lists:flatten(RawElem), - case lists:member($\n, Elem) of - true -> lists:flatten(lists:join("\n", In)); - false -> Elem ++ ", " ++ to_lines(Rest) - end. + Elem = lists:flatten(RawElem), + case lists:member($\n, Elem) of + true -> lists:flatten(lists:join("\n", In)); + false -> Elem ++ ", " ++ to_lines(Rest) + end. %% @doc Format a string with an indentation level. format_indented(Str, Indent) -> format_indented(Str, "", Indent). format_indented(RawStr, Fmt, Ind) -> - IndentSpaces = hb_opts:get(debug_print_indent), - lists:droplast( - lists:flatten( - io_lib:format( - [$\s || _ <- lists:seq(1, Ind * IndentSpaces)] ++ - lists:flatten(RawStr) ++ "\n", - Fmt - ) - ) - ). + IndentSpaces = hb_opts:get(debug_print_indent), + lists:droplast( + lists:flatten( + io_lib:format( + [$\s || _ <- lists:seq(1, Ind * IndentSpaces)] ++ + lists:flatten(RawStr) ++ "\n", + Fmt + ) + ) + ). %% @doc Format a binary as a short string suitable for printing. format_binary(Bin) -> - MaxBinPrint = hb_opts:get(debug_print_binary_max), - Printable = - binary:part( - Bin, - 0, - case byte_size(Bin) of - X when X < MaxBinPrint -> X; - _ -> MaxBinPrint - end - ), - PrintSegment = lists:flatten(io_lib:format("~p", [Printable])), - lists:flatten( - io_lib:format( - "~s~s <~p bytes>", - [ - PrintSegment, - case Bin == Printable of - true -> ""; - false -> "..." - end, - byte_size(Bin) - ] - ) - ). + MaxBinPrint = hb_opts:get(debug_print_binary_max), + Printable = + binary:part( + Bin, + 0, + case byte_size(Bin) of + X when X < MaxBinPrint -> X; + _ -> MaxBinPrint + end + ), + PrintSegment = lists:flatten(io_lib:format("~p", [Printable])), + lists:flatten( + io_lib:format( + "~s~s <~p bytes>", + [ + PrintSegment, + case Bin == Printable of + true -> ""; + false -> "..." + end, + byte_size(Bin) + ] + ) + ). %% @doc Format a map as either a single line or a multi-line string depending %% on the value of the `debug_print_map_line_threshold` runtime option. format_map(Map) -> format_map(Map, 0). format_map(Map, Indent) -> - MaxLen = hb_opts:get(debug_print_map_line_threshold), - SimpleFmt = io_lib:format("~p", [Map]), - case lists:flatlength(SimpleFmt) of - Len when Len > MaxLen -> - "\n" ++ lists:flatten(hb_message:format(Map, Indent)); - _ -> SimpleFmt - end. + MaxLen = hb_opts:get(debug_print_map_line_threshold), + SimpleFmt = io_lib:format("~p", [Map]), + case lists:flatlength(SimpleFmt) of + Len when Len > MaxLen -> + "\n" ++ lists:flatten(hb_message:format(Map, Indent)); + _ -> SimpleFmt + end. %% @doc Print the trace of the current stack, up to the first non-hyperbeam %% module. Prints each stack frame on a new line, until it finds a frame that @@ -308,22 +308,22 @@ format_map(Map, Indent) -> %% Optionally, you may call this function with a custom label and caller info, %% which will be used instead of the default. print_trace(Stack, CallMod, CallFunc, CallLine) -> - print_trace(Stack, "HB TRACE", - lists:flatten(io_lib:format("[~s:~w ~p]", - [CallMod, CallLine, CallFunc]) - )). + print_trace(Stack, "HB TRACE", + lists:flatten(io_lib:format("[~s:~w ~p]", + [CallMod, CallLine, CallFunc]) + )). print_trace(Stack, Label, CallerInfo) -> - io:format(standard_error, "=== ~s ===~s==>~n~s", - [ - Label, CallerInfo, - lists:flatten( - format_trace( - Stack, - hb_opts:get(stack_print_prefixes, [], #{}) - ) - ) - ]). + io:format(standard_error, "=== ~s ===~s==>~n~s", + [ + Label, CallerInfo, + lists:flatten( + format_trace( + Stack, + hb_opts:get(stack_print_prefixes, [], #{}) + ) + ) + ]). %% @doc Format a stack trace as a list of strings, one for each stack frame. %% Each stack frame is formatted if it matches the `stack_print_prefixes` @@ -331,44 +331,44 @@ print_trace(Stack, Label, CallerInfo) -> %% `stack_print_prefixes` option, the rest of the stack is not formatted. format_trace([], _) -> []; format_trace([Item|Rest], Prefixes) -> - case element(1, Item) of - Atom when is_atom(Atom) -> - case string:tokens(atom_to_list(Atom), "_") of - [Prefix, _] -> - case lists:member( - Prefix, - Prefixes - ) of - true -> - [ - format_trace(Item, Prefixes) | - format_trace(Rest, Prefixes) - ]; - false -> [] - end; - _ -> [] - end; - _ -> [] - end; + case element(1, Item) of + Atom when is_atom(Atom) -> + case string:tokens(atom_to_list(Atom), "_") of + [Prefix, _] -> + case lists:member( + Prefix, + Prefixes + ) of + true -> + [ + format_trace(Item, Prefixes) | + format_trace(Rest, Prefixes) + ]; + false -> [] + end; + _ -> [] + end; + _ -> [] + end; format_trace({Func, ArityOrTerm, Extras}, Prefixes) -> - format_trace({no_module, Func, ArityOrTerm, Extras}, Prefixes); + format_trace({no_module, Func, ArityOrTerm, Extras}, Prefixes); format_trace({Mod, Func, ArityOrTerm, Extras}, _Prefixes) -> - ExtraMap = maps:from_list(Extras), - format_indented( - "~p:~p/~p [~s]~n", - [ - Mod, Func, ArityOrTerm, - case maps:get(line, ExtraMap, undefined) of - undefined -> "No details"; - Line -> - maps:get(file, ExtraMap) - ++ ":" ++ integer_to_list(Line) - end - ], - 1 - ). + ExtraMap = maps:from_list(Extras), + format_indented( + "~p:~p/~p [~s]~n", + [ + Mod, Func, ArityOrTerm, + case maps:get(line, ExtraMap, undefined) of + undefined -> "No details"; + Line -> + maps:get(file, ExtraMap) + ++ ":" ++ integer_to_list(Line) + end + ], + 1 + ). %% @doc Utility function to help macro `?trace/0` remove the first frame of the %% stack trace. trace_macro_helper({_, {_, [_IgnoredFrame|Stack]}}, Mod, Func, Line) -> - print_trace(Stack, Mod, Func, Line). \ No newline at end of file + print_trace(Stack, Mod, Func, Line). \ No newline at end of file diff --git a/src/include/ar.hrl b/src/include/ar.hrl index b114b7e2..0a6c29c7 100644 --- a/src/include/ar.hrl +++ b/src/include/ar.hrl @@ -7,61 +7,61 @@ -define(MAX_TAG_VALUE_SIZE, 3072). %% @doc A transaction. -record(tx, { - %% 1 or 2 or ans104. - format = ans104, - %% The transaction identifier. - id = ?DEFAULT_ID, - unsigned_id = ?DEFAULT_ID, - %% Either the identifier of the previous transaction from - %% the same wallet or the identifier of one of the - %% last ?MAX_TX_ANCHOR_DEPTH blocks. - last_tx = <<>>, - %% The public key the transaction is signed with. - owner = ?DEFAULT_OWNER, - %% A list of arbitrary key-value pairs. Keys and values are binaries. - tags = [], - %% The address of the recipient, if any. The SHA2-256 hash of the public key. - target = <<>>, - %% The amount of Winstons to send to the recipient, if any. - quantity = 0, - %% The data to upload, if any. For v2 transactions, the field is optional - a fee - %% is charged based on the "data_size" field, data itself may be uploaded any time - %% later in chunks. - data = ?DEFAULT_DATA, - manifest = undefined, - %% Size in bytes of the transaction data. - data_size = 0, - %% Deprecated. Not used, not gossiped. - data_tree = [], - %% The Merkle root of the Merkle tree of data chunks. - data_root = <<>>, - %% The signature. - signature = ?DEFAULT_SIG, - %% The fee in Winstons. - reward = 0, + %% 1 or 2 or ans104. + format = ans104, + %% The transaction identifier. + id = ?DEFAULT_ID, + unsigned_id = ?DEFAULT_ID, + %% Either the identifier of the previous transaction from + %% the same wallet or the identifier of one of the + %% last ?MAX_TX_ANCHOR_DEPTH blocks. + last_tx = <<>>, + %% The public key the transaction is signed with. + owner = ?DEFAULT_OWNER, + %% A list of arbitrary key-value pairs. Keys and values are binaries. + tags = [], + %% The address of the recipient, if any. The SHA2-256 hash of the public key. + target = <<>>, + %% The amount of Winstons to send to the recipient, if any. + quantity = 0, + %% The data to upload, if any. For v2 transactions, the field is optional - a fee + %% is charged based on the "data_size" field, data itself may be uploaded any time + %% later in chunks. + data = ?DEFAULT_DATA, + manifest = undefined, + %% Size in bytes of the transaction data. + data_size = 0, + %% Deprecated. Not used, not gossiped. + data_tree = [], + %% The Merkle root of the Merkle tree of data chunks. + data_root = <<>>, + %% The signature. + signature = ?DEFAULT_SIG, + %% The fee in Winstons. + reward = 0, - %% The code for the denomination of AR in base units. - %% - %% 1 corresponds to the original denomination of 1^12 base units. - %% Every time the available supply falls below ?REDENOMINATION_THRESHOLD, - %% the denomination is multiplied by 1000, the code is incremented. - %% - %% 0 is the default denomination code. It is treated as the denomination code of the - %% current block. We do NOT default to 1 because we want to distinguish between the - %% transactions with the explicitly assigned denomination (the denomination then becomes - %% a part of the signature preimage) and transactions signed the way they were signed - %% before the upgrade. The motivation is to keep supporting legacy client libraries after - %% redenominations and at the same time protect users from an attack where - %% a post-redenomination transaction is included in a pre-redenomination block. The attack - %% is prevented by forbidding inclusion of transactions with denomination=0 in the 100 - %% blocks preceding the redenomination block. - %% - %% Transaction denomination code must not exceed the block's denomination code. - denomination = 0, + %% The code for the denomination of AR in base units. + %% + %% 1 corresponds to the original denomination of 1^12 base units. + %% Every time the available supply falls below ?REDENOMINATION_THRESHOLD, + %% the denomination is multiplied by 1000, the code is incremented. + %% + %% 0 is the default denomination code. It is treated as the denomination code of the + %% current block. We do NOT default to 1 because we want to distinguish between the + %% transactions with the explicitly assigned denomination (the denomination then becomes + %% a part of the signature preimage) and transactions signed the way they were signed + %% before the upgrade. The motivation is to keep supporting legacy client libraries after + %% redenominations and at the same time protect users from an attack where + %% a post-redenomination transaction is included in a pre-redenomination block. The attack + %% is prevented by forbidding inclusion of transactions with denomination=0 in the 100 + %% blocks preceding the redenomination block. + %% + %% Transaction denomination code must not exceed the block's denomination code. + denomination = 0, - %% The type of signature this transaction was signed with. A system field, - %% not used by the protocol yet. - signature_type = {rsa, 65537} + %% The type of signature this transaction was signed with. A system field, + %% not used by the protocol yet. + signature_type = {rsa, 65537} }). %% The hashing algorithm used to calculate wallet addresses. diff --git a/src/include/hb.hrl b/src/include/hb.hrl index b29c2110..99731460 100644 --- a/src/include/hb.hrl +++ b/src/include/hb.hrl @@ -21,11 +21,11 @@ -define(trace(), hb_util:trace_macro_helper(catch error(test), ?MODULE, ?FUNCTION_NAME, ?LINE)). -record(result, { - messages = [], - assignments = [], - spawns = [], - output = [], - cursor = undefined + messages = [], + assignments = [], + spawns = [], + output = [], + cursor = undefined }). -record(pstate, { diff --git a/src/include/hb_http.hrl b/src/include/hb_http.hrl index 2b11e897..eaae88ef 100644 --- a/src/include/hb_http.hrl +++ b/src/include/hb_http.hrl @@ -1,72 +1,72 @@ -define(IS_ALPHA(C), - (C =:= $a) or (C =:= $b) or (C =:= $c) or (C =:= $d) or (C =:= $e) or - (C =:= $f) or (C =:= $g) or (C =:= $h) or (C =:= $i) or (C =:= $j) or - (C =:= $k) or (C =:= $l) or (C =:= $m) or (C =:= $n) or (C =:= $o) or - (C =:= $p) or (C =:= $q) or (C =:= $r) or (C =:= $s) or (C =:= $t) or - (C =:= $u) or (C =:= $v) or (C =:= $w) or (C =:= $x) or (C =:= $y) or - (C =:= $z) or - (C =:= $A) or (C =:= $B) or (C =:= $C) or (C =:= $D) or (C =:= $E) or - (C =:= $F) or (C =:= $G) or (C =:= $H) or (C =:= $I) or (C =:= $J) or - (C =:= $K) or (C =:= $L) or (C =:= $M) or (C =:= $N) or (C =:= $O) or - (C =:= $P) or (C =:= $Q) or (C =:= $R) or (C =:= $S) or (C =:= $T) or - (C =:= $U) or (C =:= $V) or (C =:= $W) or (C =:= $X) or (C =:= $Y) or - (C =:= $Z) + (C =:= $a) or (C =:= $b) or (C =:= $c) or (C =:= $d) or (C =:= $e) or + (C =:= $f) or (C =:= $g) or (C =:= $h) or (C =:= $i) or (C =:= $j) or + (C =:= $k) or (C =:= $l) or (C =:= $m) or (C =:= $n) or (C =:= $o) or + (C =:= $p) or (C =:= $q) or (C =:= $r) or (C =:= $s) or (C =:= $t) or + (C =:= $u) or (C =:= $v) or (C =:= $w) or (C =:= $x) or (C =:= $y) or + (C =:= $z) or + (C =:= $A) or (C =:= $B) or (C =:= $C) or (C =:= $D) or (C =:= $E) or + (C =:= $F) or (C =:= $G) or (C =:= $H) or (C =:= $I) or (C =:= $J) or + (C =:= $K) or (C =:= $L) or (C =:= $M) or (C =:= $N) or (C =:= $O) or + (C =:= $P) or (C =:= $Q) or (C =:= $R) or (C =:= $S) or (C =:= $T) or + (C =:= $U) or (C =:= $V) or (C =:= $W) or (C =:= $X) or (C =:= $Y) or + (C =:= $Z) ). -define(IS_ALPHANUM(C), ?IS_ALPHA(C) or ?IS_DIGIT(C)). -define(IS_CHAR(C), C > 0, C < 128). -define(IS_DIGIT(C), - (C =:= $0) or (C =:= $1) or (C =:= $2) or (C =:= $3) or (C =:= $4) or - (C =:= $5) or (C =:= $6) or (C =:= $7) or (C =:= $8) or (C =:= $9) + (C =:= $0) or (C =:= $1) or (C =:= $2) or (C =:= $3) or (C =:= $4) or + (C =:= $5) or (C =:= $6) or (C =:= $7) or (C =:= $8) or (C =:= $9) ). -define(IS_ETAGC(C), C =:= 16#21; C >= 16#23, C =/= 16#7f). -define(IS_HEX(C), - ?IS_DIGIT(C) or - (C =:= $a) or (C =:= $b) or (C =:= $c) or - (C =:= $d) or (C =:= $e) or (C =:= $f) or - (C =:= $A) or (C =:= $B) or (C =:= $C) or - (C =:= $D) or (C =:= $E) or (C =:= $F) + ?IS_DIGIT(C) or + (C =:= $a) or (C =:= $b) or (C =:= $c) or + (C =:= $d) or (C =:= $e) or (C =:= $f) or + (C =:= $A) or (C =:= $B) or (C =:= $C) or + (C =:= $D) or (C =:= $E) or (C =:= $F) ). -define(IS_LHEX(C), - ?IS_DIGIT(C) or - (C =:= $a) or (C =:= $b) or (C =:= $c) or - (C =:= $d) or (C =:= $e) or (C =:= $f) + ?IS_DIGIT(C) or + (C =:= $a) or (C =:= $b) or (C =:= $c) or + (C =:= $d) or (C =:= $e) or (C =:= $f) ). -define(IS_TOKEN(C), - ?IS_ALPHA(C) or ?IS_DIGIT(C) or - (C =:= $!) or (C =:= $#) or (C =:= $$) or (C =:= $%) or (C =:= $&) or - (C =:= $') or (C =:= $*) or (C =:= $+) or (C =:= $-) or (C =:= $.) or - (C =:= $^) or (C =:= $_) or (C =:= $`) or (C =:= $|) or (C =:= $~) + ?IS_ALPHA(C) or ?IS_DIGIT(C) or + (C =:= $!) or (C =:= $#) or (C =:= $$) or (C =:= $%) or (C =:= $&) or + (C =:= $') or (C =:= $*) or (C =:= $+) or (C =:= $-) or (C =:= $.) or + (C =:= $^) or (C =:= $_) or (C =:= $`) or (C =:= $|) or (C =:= $~) ). -define(IS_TOKEN68(C), - ?IS_ALPHA(C) or ?IS_DIGIT(C) or - (C =:= $-) or (C =:= $.) or (C =:= $_) or - (C =:= $~) or (C =:= $+) or (C =:= $/) + ?IS_ALPHA(C) or ?IS_DIGIT(C) or + (C =:= $-) or (C =:= $.) or (C =:= $_) or + (C =:= $~) or (C =:= $+) or (C =:= $/) ). -define(IS_URI_UNRESERVED(C), - ?IS_ALPHA(C) or ?IS_DIGIT(C) or - (C =:= $-) or (C =:= $.) or (C =:= $_) or (C =:= $~) + ?IS_ALPHA(C) or ?IS_DIGIT(C) or + (C =:= $-) or (C =:= $.) or (C =:= $_) or (C =:= $~) ). -define(IS_URI_GEN_DELIMS(C), - (C =:= $:) or (C =:= $/) or (C =:= $?) or (C =:= $#) or - (C =:= $[) or (C =:= $]) or (C =:= $@) + (C =:= $:) or (C =:= $/) or (C =:= $?) or (C =:= $#) or + (C =:= $[) or (C =:= $]) or (C =:= $@) ). -define(IS_URI_SUB_DELIMS(C), - (C =:= $!) or (C =:= $$) or (C =:= $&) or (C =:= $') or - (C =:= $() or (C =:= $)) or (C =:= $*) or (C =:= $+) or - (C =:= $,) or (C =:= $;) or (C =:= $=) + (C =:= $!) or (C =:= $$) or (C =:= $&) or (C =:= $') or + (C =:= $() or (C =:= $)) or (C =:= $*) or (C =:= $+) or + (C =:= $,) or (C =:= $;) or (C =:= $=) ). -define(IS_VCHAR(C), C =:= $\t; C > 31, C < 127). -define(IS_VCHAR_OBS(C), C =:= $\t; C > 31, C =/= 127). -define(IS_WS(C), (C =:= $\s) or (C =:= $\t)). --define(IS_WS_COMMA(C), ?IS_WS(C) or (C =:= $,)). +-define(IS_WS_COMMA(C), ?IS_WS(C) or (C =:= $,)). \ No newline at end of file diff --git a/src/rsa_pss.erl b/src/rsa_pss.erl index 5a0969d2..7b4cc996 100644 --- a/src/rsa_pss.erl +++ b/src/rsa_pss.erl @@ -32,145 +32,145 @@ %%==================================================================== -spec sign(Message, DigestType, PrivateKey) -> Signature - when - Message :: binary() | {digest, binary()}, - DigestType :: rsa_digest_type() | atom(), - PrivateKey :: rsa_private_key(), - Signature :: binary(). + when + Message :: binary() | {digest, binary()}, + DigestType :: rsa_digest_type() | atom(), + PrivateKey :: rsa_private_key(), + Signature :: binary(). sign(Message, DigestType, PrivateKey) when is_binary(Message) -> - sign({digest, crypto:hash(DigestType, Message)}, DigestType, PrivateKey); + sign({digest, crypto:hash(DigestType, Message)}, DigestType, PrivateKey); sign(Message={digest, _}, DigestType, PrivateKey) -> - SaltLen = byte_size(crypto:hash(DigestType, <<>>)), - Salt = crypto:strong_rand_bytes(SaltLen), - sign(Message, DigestType, Salt, PrivateKey). + SaltLen = byte_size(crypto:hash(DigestType, <<>>)), + Salt = crypto:strong_rand_bytes(SaltLen), + sign(Message, DigestType, Salt, PrivateKey). -spec sign(Message, DigestType, Salt, PrivateKey) -> Signature - when - Message :: binary() | {digest, binary()}, - DigestType :: rsa_digest_type() | atom(), - Salt :: binary(), - PrivateKey :: rsa_private_key(), - Signature :: binary(). + when + Message :: binary() | {digest, binary()}, + DigestType :: rsa_digest_type() | atom(), + Salt :: binary(), + PrivateKey :: rsa_private_key(), + Signature :: binary(). sign(Message, DigestType, Salt, PrivateKey) when is_binary(Message) -> - sign({digest, crypto:hash(DigestType, Message)}, DigestType, Salt, PrivateKey); + sign({digest, crypto:hash(DigestType, Message)}, DigestType, Salt, PrivateKey); sign({digest, Digest}, DigestType, Salt, PrivateKey=#'RSAPrivateKey'{modulus=N}) -> - DigestLen = byte_size(Digest), - SaltLen = byte_size(Salt), - PublicBitSize = int_to_bit_size(N), - PrivateByteSize = (PublicBitSize + 7) div 8, - PublicByteSize = int_to_byte_size(N), - case PublicByteSize < (DigestLen + SaltLen + 2) of - false -> - DBLen = PrivateByteSize - DigestLen - 1, - M = << 0:64, Digest/binary, Salt/binary >>, - H = crypto:hash(DigestType, M), - DB = << 0:((DBLen - SaltLen - 1) * 8), 1, Salt/binary >>, - DBMask = mgf1(DigestType, H, DBLen), - MaskedDB = normalize_to_key_size(PublicBitSize, crypto:exor(DB, DBMask)), - EM = << MaskedDB/binary, H/binary, ?PSS_TRAILER_FIELD >>, - DM = pad_to_key_size(PublicByteSize, dp(EM, PrivateKey)), - DM; - true -> - erlang:error(badarg, [{digest, Digest}, DigestType, Salt, PrivateKey]) - end. + DigestLen = byte_size(Digest), + SaltLen = byte_size(Salt), + PublicBitSize = int_to_bit_size(N), + PrivateByteSize = (PublicBitSize + 7) div 8, + PublicByteSize = int_to_byte_size(N), + case PublicByteSize < (DigestLen + SaltLen + 2) of + false -> + DBLen = PrivateByteSize - DigestLen - 1, + M = << 0:64, Digest/binary, Salt/binary >>, + H = crypto:hash(DigestType, M), + DB = << 0:((DBLen - SaltLen - 1) * 8), 1, Salt/binary >>, + DBMask = mgf1(DigestType, H, DBLen), + MaskedDB = normalize_to_key_size(PublicBitSize, crypto:exor(DB, DBMask)), + EM = << MaskedDB/binary, H/binary, ?PSS_TRAILER_FIELD >>, + DM = pad_to_key_size(PublicByteSize, dp(EM, PrivateKey)), + DM; + true -> + erlang:error(badarg, [{digest, Digest}, DigestType, Salt, PrivateKey]) + end. -spec verify(Message, DigestType, Signature, PublicKey) -> boolean() - when - Message :: binary() | {digest, binary()}, - DigestType :: rsa_digest_type() | atom(), - Signature :: binary(), - PublicKey :: rsa_public_key(). + when + Message :: binary() | {digest, binary()}, + DigestType :: rsa_digest_type() | atom(), + Signature :: binary(), + PublicKey :: rsa_public_key(). verify(Message, DigestType, Signature, PublicKey) when is_binary(Message) -> - verify({digest, crypto:hash(DigestType, Message)}, DigestType, Signature, PublicKey); + verify({digest, crypto:hash(DigestType, Message)}, DigestType, Signature, PublicKey); verify({digest, Digest}, DigestType, Signature, PublicKey=#'RSAPublicKey'{modulus=N}) -> - DigestLen = byte_size(Digest), - PublicBitSize = int_to_bit_size(N), - PrivateByteSize = (PublicBitSize + 7) div 8, - PublicByteSize = int_to_byte_size(N), - SignatureSize = byte_size(Signature), - case PublicByteSize =:= SignatureSize of - true -> - SignatureNumber = binary:decode_unsigned(Signature, big), - case SignatureNumber >= 0 andalso SignatureNumber < N of - true -> - DBLen = PrivateByteSize - DigestLen - 1, - EM = pad_to_key_size(PrivateByteSize, ep(Signature, PublicKey)), - case binary:last(EM) of - ?PSS_TRAILER_FIELD -> - MaskedDB = binary:part(EM, 0, byte_size(EM) - DigestLen - 1), - H = binary:part(EM, byte_size(MaskedDB), DigestLen), - DBMask = mgf1(DigestType, H, DBLen), - DB = normalize_to_key_size(PublicBitSize, crypto:exor(MaskedDB, DBMask)), - case binary:match(DB, << 1 >>) of - {Pos, Len} -> - PS = binary:decode_unsigned(binary:part(DB, 0, Pos)), - case PS =:= 0 of - true -> - Salt = binary:part(DB, Pos + Len, byte_size(DB) - Pos - Len), - M = << 0:64, Digest/binary, Salt/binary >>, - HOther = crypto:hash(DigestType, M), - H =:= HOther; - false -> - false - end; - nomatch -> - false - end; - _BadTrailer -> - false - end; - _ -> - false - end; - false -> - false - end. + DigestLen = byte_size(Digest), + PublicBitSize = int_to_bit_size(N), + PrivateByteSize = (PublicBitSize + 7) div 8, + PublicByteSize = int_to_byte_size(N), + SignatureSize = byte_size(Signature), + case PublicByteSize =:= SignatureSize of + true -> + SignatureNumber = binary:decode_unsigned(Signature, big), + case SignatureNumber >= 0 andalso SignatureNumber < N of + true -> + DBLen = PrivateByteSize - DigestLen - 1, + EM = pad_to_key_size(PrivateByteSize, ep(Signature, PublicKey)), + case binary:last(EM) of + ?PSS_TRAILER_FIELD -> + MaskedDB = binary:part(EM, 0, byte_size(EM) - DigestLen - 1), + H = binary:part(EM, byte_size(MaskedDB), DigestLen), + DBMask = mgf1(DigestType, H, DBLen), + DB = normalize_to_key_size(PublicBitSize, crypto:exor(MaskedDB, DBMask)), + case binary:match(DB, << 1 >>) of + {Pos, Len} -> + PS = binary:decode_unsigned(binary:part(DB, 0, Pos)), + case PS =:= 0 of + true -> + Salt = binary:part(DB, Pos + Len, byte_size(DB) - Pos - Len), + M = << 0:64, Digest/binary, Salt/binary >>, + HOther = crypto:hash(DigestType, M), + H =:= HOther; + false -> + false + end; + nomatch -> + false + end; + _BadTrailer -> + false + end; + _ -> + false + end; + false -> + false + end. verify_legacy(Message, DigestType, Signature, PublicKey) when is_binary(Message) -> - verify_legacy({digest, crypto:hash(DigestType, Message)}, DigestType, Signature, PublicKey); + verify_legacy({digest, crypto:hash(DigestType, Message)}, DigestType, Signature, PublicKey); verify_legacy({digest, Digest}, DigestType, Signature, PublicKey=#'RSAPublicKey'{modulus=N}) -> - DigestLen = byte_size(Digest), - PublicBitSize = int_to_bit_size(N), - PrivateByteSize = PublicBitSize div 8, - PublicByteSize = int_to_byte_size(N), - SignatureSize = byte_size(Signature), - case PublicByteSize =:= SignatureSize of - true -> - SignatureNumber = binary:decode_unsigned(Signature, big), - case SignatureNumber >= 0 andalso SignatureNumber < N of - true -> - DBLen = PrivateByteSize - DigestLen - 1, - EM = pad_to_key_size(PrivateByteSize, ep(Signature, PublicKey)), - case binary:last(EM) of - ?PSS_TRAILER_FIELD -> - MaskedDB = binary:part(EM, 0, byte_size(EM) - DigestLen - 1), - H = binary:part(EM, byte_size(MaskedDB), DigestLen), - DBMask = mgf1(DigestType, H, DBLen), - DB = normalize_to_key_size(PublicBitSize, crypto:exor(MaskedDB, DBMask)), - case binary:match(DB, << 1 >>) of - {Pos, Len} -> - PS = binary:decode_unsigned(binary:part(DB, 0, Pos)), - case PS =:= 0 of - true -> - Salt = binary:part(DB, Pos + Len, byte_size(DB) - Pos - Len), - M = << 0:64, Digest/binary, Salt/binary >>, - HOther = crypto:hash(DigestType, M), - H =:= HOther; - false -> - false - end; - nomatch -> - false - end; - _BadTrailer -> - false - end; - _ -> - false - end; - false -> - false - end. + DigestLen = byte_size(Digest), + PublicBitSize = int_to_bit_size(N), + PrivateByteSize = PublicBitSize div 8, + PublicByteSize = int_to_byte_size(N), + SignatureSize = byte_size(Signature), + case PublicByteSize =:= SignatureSize of + true -> + SignatureNumber = binary:decode_unsigned(Signature, big), + case SignatureNumber >= 0 andalso SignatureNumber < N of + true -> + DBLen = PrivateByteSize - DigestLen - 1, + EM = pad_to_key_size(PrivateByteSize, ep(Signature, PublicKey)), + case binary:last(EM) of + ?PSS_TRAILER_FIELD -> + MaskedDB = binary:part(EM, 0, byte_size(EM) - DigestLen - 1), + H = binary:part(EM, byte_size(MaskedDB), DigestLen), + DBMask = mgf1(DigestType, H, DBLen), + DB = normalize_to_key_size(PublicBitSize, crypto:exor(MaskedDB, DBMask)), + case binary:match(DB, << 1 >>) of + {Pos, Len} -> + PS = binary:decode_unsigned(binary:part(DB, 0, Pos)), + case PS =:= 0 of + true -> + Salt = binary:part(DB, Pos + Len, byte_size(DB) - Pos - Len), + M = << 0:64, Digest/binary, Salt/binary >>, + HOther = crypto:hash(DigestType, M), + H =:= HOther; + false -> + false + end; + nomatch -> + false + end; + _BadTrailer -> + false + end; + _ -> + false + end; + false -> + false + end. %%%------------------------------------------------------------------- %%% Internal functions @@ -178,60 +178,60 @@ verify_legacy({digest, Digest}, DigestType, Signature, PublicKey=#'RSAPublicKey' %% @private dp(B, #'RSAPrivateKey'{modulus=N, privateExponent=E}) -> - crypto:mod_pow(B, E, N). + crypto:mod_pow(B, E, N). %% @private ep(B, #'RSAPublicKey'{modulus=N, publicExponent=E}) -> - crypto:mod_pow(B, E, N). + crypto:mod_pow(B, E, N). %% @private int_to_bit_size(I) -> - int_to_bit_size(I, 0). + int_to_bit_size(I, 0). %% @private int_to_bit_size(0, B) -> - B; + B; int_to_bit_size(I, B) -> - int_to_bit_size(I bsr 1, B + 1). + int_to_bit_size(I bsr 1, B + 1). %% @private int_to_byte_size(I) -> - int_to_byte_size(I, 0). + int_to_byte_size(I, 0). %% @private int_to_byte_size(0, B) -> - B; + B; int_to_byte_size(I, B) -> - int_to_byte_size(I bsr 8, B + 1). + int_to_byte_size(I bsr 8, B + 1). %% @private mgf1(DigestType, Seed, Len) -> - mgf1(DigestType, Seed, Len, <<>>, 0). + mgf1(DigestType, Seed, Len, <<>>, 0). %% @private mgf1(_DigestType, _Seed, Len, T, _Counter) when byte_size(T) >= Len -> - binary:part(T, 0, Len); + binary:part(T, 0, Len); mgf1(DigestType, Seed, Len, T, Counter) -> - CounterBin = << Counter:8/unsigned-big-integer-unit:4 >>, - NewT = << T/binary, (crypto:hash(DigestType, << Seed/binary, CounterBin/binary >>))/binary >>, - mgf1(DigestType, Seed, Len, NewT, Counter + 1). + CounterBin = << Counter:8/unsigned-big-integer-unit:4 >>, + NewT = << T/binary, (crypto:hash(DigestType, << Seed/binary, CounterBin/binary >>))/binary >>, + mgf1(DigestType, Seed, Len, NewT, Counter + 1). %% @private normalize_to_key_size(_, <<>>) -> - <<>>; + <<>>; normalize_to_key_size(Bits, _A = << C, Rest/binary >>) -> - SH = (Bits - 1) band 16#7, - Mask = case SH > 0 of - false -> - 16#FF; - true -> - 16#FF bsr (8 - SH) - end, - B = << (C band Mask), Rest/binary >>, - B. + SH = (Bits - 1) band 16#7, + Mask = case SH > 0 of + false -> + 16#FF; + true -> + 16#FF bsr (8 - SH) + end, + B = << (C band Mask), Rest/binary >>, + B. %% @private pad_to_key_size(Bytes, Data) when byte_size(Data) < Bytes -> - pad_to_key_size(Bytes, << 0, Data/binary >>); + pad_to_key_size(Bytes, << 0, Data/binary >>); pad_to_key_size(_Bytes, Data) -> - Data. \ No newline at end of file + Data. \ No newline at end of file diff --git a/src/sec.erl b/src/sec.erl index cf07b66c..291de1f3 100644 --- a/src/sec.erl +++ b/src/sec.erl @@ -33,69 +33,69 @@ %% Generate attestation based on the provided nonce (both TPM and SEV-SNP) generate_attestation(Nonce) -> - ?event({"Generating TPM attestation..."}), + ?event({"Generating TPM attestation..."}), - case sec_tpm:generate_attestation(Nonce) of - {ok, TPMAttestation} -> - ?event({"TPM attestation generated, size:", byte_size(TPMAttestation)}), + case sec_tpm:generate_attestation(Nonce) of + {ok, TPMAttestation} -> + ?event({"TPM attestation generated, size:", byte_size(TPMAttestation)}), - ?event({"Generating SEV-SNP attestation..."}), + ?event({"Generating SEV-SNP attestation..."}), - case sec_tee:generate_attestation(Nonce) of - {ok, TEEAttestation} -> - ?event({"SEV-SNP attestation generated, size:", byte_size(TEEAttestation)}), + case sec_tee:generate_attestation(Nonce) of + {ok, TEEAttestation} -> + ?event({"SEV-SNP attestation generated, size:", byte_size(TEEAttestation)}), - %% Calculate sizes of the two attestation binaries - TPMSize = byte_size(TPMAttestation), - TEESize = byte_size(TEEAttestation), + %% Calculate sizes of the two attestation binaries + TPMSize = byte_size(TPMAttestation), + TEESize = byte_size(TEEAttestation), - %% Create the header containing the sizes - Header = <>, - ?event({"Header created, TPMSize:", TPMSize, "TEESize:", TEESize}), + %% Create the header containing the sizes + Header = <>, + ?event({"Header created, TPMSize:", TPMSize, "TEESize:", TEESize}), - %% Combine the header with the two attestation binaries - CombinedAttestation = <
>, - ?event({"Combined attestation binary created, total size:", byte_size(CombinedAttestation)}), + %% Combine the header with the two attestation binaries + CombinedAttestation = <
>, + ?event({"Combined attestation binary created, total size:", byte_size(CombinedAttestation)}), - {ok, CombinedAttestation}; - {error, Reason} -> - ?event({"Error generating SEV-SNP attestation:", Reason}), - {error, Reason} - end; - {error, Reason} -> - ?event({"Error generating TPM attestation:", Reason}), - {error, Reason} - end. + {ok, CombinedAttestation}; + {error, Reason} -> + ?event({"Error generating SEV-SNP attestation:", Reason}), + {error, Reason} + end; + {error, Reason} -> + ?event({"Error generating TPM attestation:", Reason}), + {error, Reason} + end. %% Verify attestation report based on the provided binary (both TPM and SEV-SNP) verify_attestation(AttestationBinary) -> - ?event("Verifying attestation..."), - - %% Extract the header (size info) and the attestation binaries - <> = AttestationBinary, - ?event({"Header extracted, TPMSize:", TPMSize, "TEESize:", TEESize}), - - %% Extract the TPM and SEV-SNP attestation binaries based on their sizes - <> = Rest, - ?event({"Extracted TPM and SEV-SNP attestation binaries"}), - - %% Verify TPM attestation - case sec_tpm:verify_attestation(TPMAttestation) of - {ok, _TPMVerification} -> - ?event({"TPM attestation verification completed"}), - - %% Verify SEV-SNP attestation - case sec_tee:verify_attestation(TEEAttestation) of - {ok, _TEEVerification} -> - ?event({"SEV-SNP attestation verification completed"}), - - %% Return success if both verifications succeeded - {ok, "Verified"}; - {error, Reason} -> - ?event({"Error verifying SEV-SNP attestation:", Reason}), - {error, Reason} - end; - {error, Reason} -> - ?event({"Error verifying TPM attestation:", Reason}), - {error, Reason} - end. + ?event("Verifying attestation..."), + + %% Extract the header (size info) and the attestation binaries + <> = AttestationBinary, + ?event({"Header extracted, TPMSize:", TPMSize, "TEESize:", TEESize}), + + %% Extract the TPM and SEV-SNP attestation binaries based on their sizes + <> = Rest, + ?event({"Extracted TPM and SEV-SNP attestation binaries"}), + + %% Verify TPM attestation + case sec_tpm:verify_attestation(TPMAttestation) of + {ok, _TPMVerification} -> + ?event({"TPM attestation verification completed"}), + + %% Verify SEV-SNP attestation + case sec_tee:verify_attestation(TEEAttestation) of + {ok, _TEEVerification} -> + ?event({"SEV-SNP attestation verification completed"}), + + %% Return success if both verifications succeeded + {ok, "Verified"}; + {error, Reason} -> + ?event({"Error verifying SEV-SNP attestation:", Reason}), + {error, Reason} + end; + {error, Reason} -> + ?event({"Error verifying TPM attestation:", Reason}), + {error, Reason} + end. \ No newline at end of file diff --git a/src/sec_helpers.erl b/src/sec_helpers.erl index ac45cca4..e28b69c2 100644 --- a/src/sec_helpers.erl +++ b/src/sec_helpers.erl @@ -6,27 +6,27 @@ %% Helper function to write data to a file write_to_file(FilePath, Data) -> - case file:write_file(FilePath, Data) of - ok -> ?event({"Written data to file", FilePath}); - {error, Reason} -> ?event({"Failed to write to file", FilePath, Reason}) - end. + case file:write_file(FilePath, Data) of + ok -> ?event({"Written data to file", FilePath}); + {error, Reason} -> ?event({"Failed to write to file", FilePath, Reason}) + end. %% Helper function to read a file read_file(FilePath) -> - ?event({"Reading file", FilePath}), - case file:read_file(FilePath) of - {ok, Data} -> {FilePath, Data}; - {error, Reason} -> {error, Reason} - end. + ?event({"Reading file", FilePath}), + case file:read_file(FilePath) of + {ok, Data} -> {FilePath, Data}; + {error, Reason} -> {error, Reason} + end. %% Generalized function to run a shell command and optionally apply a success function %% When SuccessFun is provided, it is called upon successful execution run_command(Command) -> - ?event({"Executing command", Command}), - Output = os:cmd(Command ++ " 2>&1"), - case Output of - % Empty output interpreted as success if no output is expected - "" -> {ok, []}; - % Return output for further inspection - _ -> {ok, Output} - end. + ?event({"Executing command", Command}), + Output = os:cmd(Command ++ " 2>&1"), + case Output of + % Empty output interpreted as success if no output is expected + "" -> {ok, []}; + % Return output for further inspection + _ -> {ok, Output} + end. \ No newline at end of file diff --git a/src/sec_tee.erl b/src/sec_tee.erl index 59510439..472880a6 100644 --- a/src/sec_tee.erl +++ b/src/sec_tee.erl @@ -41,7 +41,7 @@ %% Temporarily hard-code the VCEK download command -define(DOWNLOAD_VCEK_CMD, - "curl --proto \'=https\' --tlsv1.2 -sSf https://kdsintf.amd.com/vcek/v1/Milan/cert_chain -o " ++ ?CERT_CHAIN_FILE + "curl --proto \'=https\' --tlsv1.2 -sSf https://kdsintf.amd.com/vcek/v1/Milan/cert_chain -o " ++ ?CERT_CHAIN_FILE ). %% Generate attestation, request certificates, download VCEK, and upload a transaction @@ -204,4 +204,4 @@ download_vcek_cert() -> {error, Reason} -> ?event({"Failed to download VCEK certificate", Reason}), {error, failed_to_download_cert} - end. + end. \ No newline at end of file diff --git a/src/sec_tpm.erl b/src/sec_tpm.erl index 09b8b997..c8dd9dc4 100644 --- a/src/sec_tpm.erl +++ b/src/sec_tpm.erl @@ -38,140 +38,140 @@ %% Define the TPM2 commands using the defined file paths -define(TPM2_CREATEPRIMARY_CMD, "tpm2_createprimary -C e -g sha256 -G rsa -c " ++ ?PRIMARY_CTX_FILE). -define(TPM2_CREATE_CMD, - "tpm2_create -C " ++ ?PRIMARY_CTX_FILE ++ " -G rsa -u " ++ ?AK_PUB_FILE ++ " -r " ++ ?AK_PRIV_FILE + "tpm2_create -C " ++ ?PRIMARY_CTX_FILE ++ " -G rsa -u " ++ ?AK_PUB_FILE ++ " -r " ++ ?AK_PRIV_FILE ). -define(TPM2_LOAD_CMD, - "tpm2_load -C " ++ ?PRIMARY_CTX_FILE ++ " -u " ++ ?AK_PUB_FILE ++ " -r " ++ ?AK_PRIV_FILE ++ " -c " ++ ?AK_CTX_FILE + "tpm2_load -C " ++ ?PRIMARY_CTX_FILE ++ " -u " ++ ?AK_PUB_FILE ++ " -r " ++ ?AK_PRIV_FILE ++ " -c " ++ ?AK_CTX_FILE ). -define(TPM2_READPUBLIC_CMD, "tpm2_readpublic -c " ++ ?AK_CTX_FILE ++ " -o " ++ ?AK_PUB_FILE ++ " -f pem"). -define(TPM2_QUOTE_CMD, - "tpm2_quote -Q --key-context " ++ ?AK_CTX_FILE ++ " -l sha256:0,1 --message " ++ ?QUOTE_MSG_FILE ++ " --signature " ++ - ?QUOTE_SIG_FILE ++ " --qualification " + "tpm2_quote -Q --key-context " ++ ?AK_CTX_FILE ++ " -l sha256:0,1 --message " ++ ?QUOTE_MSG_FILE ++ " --signature " ++ + ?QUOTE_SIG_FILE ++ " --qualification " ). -define(TPM2_VERIFY_CMD, - "tpm2_verifysignature -c " ++ ?AK_CTX_FILE ++ " -g sha256 -m " ++ ?QUOTE_MSG_FILE ++ " -s " ++ ?QUOTE_SIG_FILE + "tpm2_verifysignature -c " ++ ?AK_CTX_FILE ++ " -g sha256 -m " ++ ?QUOTE_MSG_FILE ++ " -s " ++ ?QUOTE_SIG_FILE ). %% Generate an attestation using the provided nonce (as a string) generate_attestation(Nonce) -> - % Check if the root directory exists, and create it if not - case filelib:is_dir(?ROOT_DIR) of - true -> ok; - false -> file:make_dir(?ROOT_DIR) - end, - - % Setup the keys - ok = setup_keys(), - - % Use the address in hex format as the nonce - NonceHex = binary_to_list(binary:encode_hex(Nonce)), - Command = ?TPM2_QUOTE_CMD ++ NonceHex, - ?event({"Running command", Command}), - case sec_helpers:run_command(Command) of - {ok, _} -> - % Read the quote.msg, quote.sig, and ak.pub files - {_, QuoteBin} = sec_helpers:read_file(?QUOTE_MSG_FILE), - {_, SignatureBin} = sec_helpers:read_file(?QUOTE_SIG_FILE), - {_, AkPubBin} = sec_helpers:read_file(?AK_PUB_FILE), - - % Get sizes of the individual files (in binary) - QuoteSize = byte_size(QuoteBin), - SignatureSize = byte_size(SignatureBin), - AkPubSize = byte_size(AkPubBin), - - % Create a binary header with the sizes and offsets - Header = <>, - - % Create a binary with all three files' data concatenated after the header - AttestationBinary = <
>, - - % Log the data (if you need to debug) - ?event({"Attestation generated, binary data:", AttestationBinary}), - - % Return the binary containing the header and files - {ok, AttestationBinary}; - {error, Reason} -> - ?event({"Error generating attestation", Reason}), - {error, Reason} - end. + % Check if the root directory exists, and create it if not + case filelib:is_dir(?ROOT_DIR) of + true -> ok; + false -> file:make_dir(?ROOT_DIR) + end, + + % Setup the keys + ok = setup_keys(), + + % Use the address in hex format as the nonce + NonceHex = binary_to_list(binary:encode_hex(Nonce)), + Command = ?TPM2_QUOTE_CMD ++ NonceHex, + ?event({"Running command", Command}), + case sec_helpers:run_command(Command) of + {ok, _} -> + % Read the quote.msg, quote.sig, and ak.pub files + {_, QuoteBin} = sec_helpers:read_file(?QUOTE_MSG_FILE), + {_, SignatureBin} = sec_helpers:read_file(?QUOTE_SIG_FILE), + {_, AkPubBin} = sec_helpers:read_file(?AK_PUB_FILE), + + % Get sizes of the individual files (in binary) + QuoteSize = byte_size(QuoteBin), + SignatureSize = byte_size(SignatureBin), + AkPubSize = byte_size(AkPubBin), + + % Create a binary header with the sizes and offsets + Header = <>, + + % Create a binary with all three files' data concatenated after the header + AttestationBinary = <
>, + + % Log the data (if you need to debug) + ?event({"Attestation generated, binary data:", AttestationBinary}), + + % Return the binary containing the header and files + {ok, AttestationBinary}; + {error, Reason} -> + ?event({"Error generating attestation", Reason}), + {error, Reason} + end. %% Verify the attestation using the AttestationBinary verify_attestation(AttestationBinary) -> - % Extract the header (size info) - <> = AttestationBinary, - - % Extract the individual components using the sizes from the header - <> = Rest, - <> = Rest1, - <> = Rest2, - - % Write the components to temporary files (if needed for verification) - sec_helpers:write_to_file(?QUOTE_MSG_FILE, QuoteData), - sec_helpers:write_to_file(?QUOTE_SIG_FILE, SignatureData), - sec_helpers:write_to_file(?AK_PUB_FILE, AkPubData), - - % Run the TPM verification command using the files - CommandVerify = ?TPM2_VERIFY_CMD, - ?event({"Running command", CommandVerify}), - case sec_helpers:run_command(CommandVerify) of - {ok, VerificationMessage} -> - {ok, VerificationMessage}; - {error, {failed, _}} = Error -> - ?event({"Verification failed", Error}), - Error - end. + % Extract the header (size info) + <> = AttestationBinary, + + % Extract the individual components using the sizes from the header + <> = Rest, + <> = Rest1, + <> = Rest2, + + % Write the components to temporary files (if needed for verification) + sec_helpers:write_to_file(?QUOTE_MSG_FILE, QuoteData), + sec_helpers:write_to_file(?QUOTE_SIG_FILE, SignatureData), + sec_helpers:write_to_file(?AK_PUB_FILE, AkPubData), + + % Run the TPM verification command using the files + CommandVerify = ?TPM2_VERIFY_CMD, + ?event({"Running command", CommandVerify}), + case sec_helpers:run_command(CommandVerify) of + {ok, VerificationMessage} -> + {ok, VerificationMessage}; + {error, {failed, _}} = Error -> + ?event({"Verification failed", Error}), + Error + end. %% Set up primary and attestation keys setup_keys() -> - ?event("Starting key setup..."), - create_primary_key(), - create_attestation_key(). + ?event("Starting key setup..."), + create_primary_key(), + create_attestation_key(). create_primary_key() -> - CommandPrimary = ?TPM2_CREATEPRIMARY_CMD, - ?event({"Running command", CommandPrimary}), - case sec_helpers:run_command(CommandPrimary) of - {ok, _} -> - ?event("Primary key created successfully"), - create_attestation_key(); - {error, Reason} -> - ?event({"Error creating primary key", Reason}), - {error, Reason} - end. + CommandPrimary = ?TPM2_CREATEPRIMARY_CMD, + ?event({"Running command", CommandPrimary}), + case sec_helpers:run_command(CommandPrimary) of + {ok, _} -> + ?event("Primary key created successfully"), + create_attestation_key(); + {error, Reason} -> + ?event({"Error creating primary key", Reason}), + {error, Reason} + end. create_attestation_key() -> - CommandCreateAK = ?TPM2_CREATE_CMD, - ?event({"Running command", CommandCreateAK}), - case sec_helpers:run_command(CommandCreateAK) of - {ok, _} -> - ?event("Attestation key created successfully"), - load_attestation_key(); - {error, Reason} -> - ?event({"Error creating attestation key", Reason}), - {error, Reason} - end. + CommandCreateAK = ?TPM2_CREATE_CMD, + ?event({"Running command", CommandCreateAK}), + case sec_helpers:run_command(CommandCreateAK) of + {ok, _} -> + ?event("Attestation key created successfully"), + load_attestation_key(); + {error, Reason} -> + ?event({"Error creating attestation key", Reason}), + {error, Reason} + end. load_attestation_key() -> - CommandLoadAK = ?TPM2_LOAD_CMD, - ?event({"Running command", CommandLoadAK}), - case sec_helpers:run_command(CommandLoadAK) of - {ok, _} -> - ?event("Attestation key loaded successfully"), - export_ak_public_key(); - {error, Reason} -> - ?event({"Error loading attestation key", Reason}), - {error, Reason} - end. + CommandLoadAK = ?TPM2_LOAD_CMD, + ?event({"Running command", CommandLoadAK}), + case sec_helpers:run_command(CommandLoadAK) of + {ok, _} -> + ?event("Attestation key loaded successfully"), + export_ak_public_key(); + {error, Reason} -> + ?event({"Error loading attestation key", Reason}), + {error, Reason} + end. %% Helper to export the AK public key export_ak_public_key() -> - Command = ?TPM2_READPUBLIC_CMD, - ?event({"Running command", Command}), - case sec_helpers:run_command(Command) of - {ok, _} -> - ?event("AK public key exported successfully"), - ok; - {error, Reason} -> - ?event({"Error exporting AK public key", Reason}), - {error, Reason} - end. + Command = ?TPM2_READPUBLIC_CMD, + ?event({"Running command", Command}), + case sec_helpers:run_command(Command) of + {ok, _} -> + ?event("AK public key exported successfully"), + ok; + {error, Reason} -> + ?event({"Error exporting AK public key", Reason}), + {error, Reason} + end. \ No newline at end of file