diff --git a/avro101/src/main/resources/0/schema1.json b/avro101/src/main/resources/0/schema1.json deleted file mode 100644 index d0ea3fef..00000000 --- a/avro101/src/main/resources/0/schema1.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "type": "record", - "name": "person", - "fields": [ - { - "name": "name", - "type": "string" - }, - { - "name": "age", - "type": [ - "int", - "null" - ] - } - ] -} diff --git a/avro101/src/main/resources/0/schema2.json b/avro101/src/main/resources/0/schema2.json deleted file mode 100644 index c17a5baf..00000000 --- a/avro101/src/main/resources/0/schema2.json +++ /dev/null @@ -1,34 +0,0 @@ -{ - "type": "record", - "name": "person2", - "fields": [ - { - "name": "name", - "type": "string" - }, - { - "name": "age", - "type": [ - "int", - "null" - ] - }, - { - "name": "address", - "type": { - "type": "record", - "name": "address", - "fields": [ - { - "name": "street", - "type": "string" - }, - { - "name": "house", - "type": "long" - } - ] - } - } - ] -} diff --git a/avro101/src/main/resources/0/value1.json b/avro101/src/main/resources/0/value1.json deleted file mode 100644 index bada64c6..00000000 --- a/avro101/src/main/resources/0/value1.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "name": "Jackery", - "age": 66, - "address": { - "street": "Broadway", - "house": 123 - } -} \ No newline at end of file diff --git a/avro101/src/main/resources/al/data1_events_key_62.json b/avro101/src/main/resources/al/data1_events_key_62.json deleted file mode 100644 index 5208dcae..00000000 --- a/avro101/src/main/resources/al/data1_events_key_62.json +++ /dev/null @@ -1,3 +0,0 @@ -{ - "mve_gkey": 1397717923 -} \ No newline at end of file diff --git a/avro101/src/main/resources/al/data2_events_value_80.json b/avro101/src/main/resources/al/data2_events_value_80.json deleted file mode 100644 index c5f18bf2..00000000 --- a/avro101/src/main/resources/al/data2_events_value_80.json +++ /dev/null @@ -1,155 +0,0 @@ -{ - "before": null, - "after": { - "tos.N4_OWN.INV_MOVE_EVENT_MV.Value": { - "berth": null, - "carrier_gkey": null, - "category": { - "string": "IMPRT" - }, - "che_carry": null, - "che_carry_login_name": null, - "che_fetch": null, - "che_fetch_login_name": null, - "che_put": null, - "che_put_login_name": null, - "che_qc": null, - "dist_carry": { - "long": 0 - }, - "dist_start": { - "long": 0 - }, - "exclude": { - "boolean": false - }, - "fm_pos_bin": { - "long": 56573944 - }, - "fm_pos_loc_gkey": { - "long": 47076 - }, - "fm_pos_locid": { - "string": "SMT" - }, - "fm_pos_loctype": "YARD", - "fm_pos_name": { - "string": "Y-SMT-184K6A" - }, - "fm_pos_orientation": { - "string": "U" - }, - "fm_pos_orientation_degrees": null, - "fm_pos_slot": { - "string": "184K6.A" - }, - "fm_pos_tier": { - "long": 1 - }, - "freight_kind": { - "string": "FCL" - }, - "line_op": { - "long": 898190 - }, - "move_kind": { - "string": "YARD" - }, - "mve_gkey": 1397717923, - "pow": { - "string": "YD_SHIFT_1" - }, - "processed": null, - "rehandle_count": { - "long": 0 - }, - "restow_account": null, - "restow_reason": null, - "service_order": null, - "to_pos_bin": { - "long": 56573943 - }, - "to_pos_locid": { - "string": "SMT" - }, - "to_pos_loctype": "YARD", - "to_pos_loc_gkey": { - "long": 47076 - }, - "to_pos_name": { - "string": "Y-SMT-184K5B" - }, - "to_pos_orientation": { - "string": "Y" - }, - "to_pos_orientation_degrees": null, - "to_pos_slot": { - "string": "184K5.B" - }, - "to_pos_tier": { - "long": 2 - }, - "twin_carry": { - "boolean": false - }, - "twin_fetch": { - "boolean": false - }, - "twin_put": { - "boolean": false - }, - "t_carry_complete": { - "long": 1668495712000000 - }, - "t_carry_dispatch": { - "long": 1668495712000000 - }, - "t_carry_fetch_ready": { - "long": 1668495712000000 - }, - "t_carry_put_ready": { - "long": 1668495712000000 - }, - "t_discharge": null, - "t_dispatch": { - "long": 1668495712000000 - }, - "t_fetch": { - "long": 1668495712000000 - }, - "t_put": { - "long": 1668495712000000 - }, - "t_tz_arrival": null, - "ufv_gkey": 1269889047 - } - }, - "source": { - "version": "1.6.0-SNAPSHOT", - "connector": "oracle", - "name": "tos", - "ts_ms": 1668495714000, - "snapshot": { - "string": "false" - }, - "db": "NN4DSMT_AVLINO", - "sequence": null, - "schema": "N4_OWN", - "table": "INV_MOVE_EVENT_MV", - "txId": { - "string": "01001800edb10a00" - }, - "scn": { - "string": "1068882541" - }, - "commit_scn": { - "string": "1068882555" - }, - "lcr_position": null - }, - "op": "c", - "ts_ms": { - "long": 1668513722322 - }, - "transaction": null -} diff --git a/avro101/src/main/resources/al/data3_bins_key_57.json b/avro101/src/main/resources/al/data3_bins_key_57.json deleted file mode 100644 index b1c29134..00000000 --- a/avro101/src/main/resources/al/data3_bins_key_57.json +++ /dev/null @@ -1,3 +0,0 @@ -{ - "gkey": 1372101433 -} \ No newline at end of file diff --git a/avro101/src/main/resources/al/data4_bins_value_65.json b/avro101/src/main/resources/al/data4_bins_value_65.json deleted file mode 100644 index 124be313..00000000 --- a/avro101/src/main/resources/al/data4_bins_value_65.json +++ /dev/null @@ -1,193 +0,0 @@ -{ - "before": null, - "after": { - "tos.N4_OWN.SPATIAL_BINS_MV.Value": { - "adjacent_block_gkey": null, - "allows_20s": { - "boolean": true - }, - "allows_24s": { - "boolean": true - }, - "allows_30s": { - "boolean": true - }, - "allows_35s": { - "boolean": true - }, - "allows_40s": { - "boolean": true - }, - "allows_45s": { - "boolean": true - }, - "allows_48s": { - "boolean": true - }, - "allows_53s": { - "boolean": true - }, - "anchor_geodetic_orientation": null, - "anchor_latitude": null, - "anchor_longitude": null, - "angle_rotation": null, - "bin_level": { - "long": 4 - }, - "bin_subclass": "YST", - "bin_type": { - "long": 52930 - }, - "block_group": null, - "block_type_enum": null, - "changed": { - "long": 1657889025552000 - }, - "changer": { - "string": "server" - }, - "che_kind_1": null, - "che_kind_2": null, - "code_chars": null, - "column_index": { - "long": 190 - }, - "compress_scan": null, - "conv_gkey_row_alt": null, - "conv_gkey_row_std": null, - "conv_gkey_tier": null, - "conv_gkey_tier_csn": null, - "coord_col_conv_id": null, - "coord_col_conv_id_40": null, - "created": { - "long": 1657889025552000 - }, - "creator": { - "string": "server" - }, - "default_col_spacing_cm": null, - "default_col_width_cm": null, - "default_doors": null, - "default_row_spacing_cm": null, - "default_row_width_cm": null, - "direction": null, - "display_order_index": null, - "dynamic_traffic_enum": null, - "extend_x_20s": null, - "extend_x_24s": null, - "extend_x_30s": null, - "extend_x_35s": null, - "extend_x_40s": null, - "extend_x_45s": null, - "extend_x_48s": null, - "extend_x_53s": null, - "facing_direction_deg": null, - "forties_extend_ascending": null, - "geometry_rotation": null, - "gkey": 1372101433, - "grid_extends_east": null, - "grid_extends_north": null, - "grid_rotation": null, - "groove": null, - "has_centerline_stack": null, - "icon_id": null, - "index_within_track": null, - "internal_pos_name": { - "string": "163G3" - }, - "internal_pos_name_alt": { - "string": "164G3" - }, - "invert_scan_x": null, - "invert_scan_y": null, - "invert_x": null, - "invert_y": null, - "is_access_blocked_first_col": null, - "is_access_blocked_last_col": null, - "label_scheme_col_only": null, - "label_scheme_full_position": null, - "label_scheme_host": null, - "label_scheme_row_only": null, - "label_scheme_tier_only": null, - "lcg_mode_enum": null, - "lcg_ref_enum": null, - "lengths_allowed": null, - "level_logical_index": null, - "lid_index_first": null, - "lid_index_last": null, - "life_cycle_state": "ACT", - "long_name": null, - "map_image": null, - "map_image_update_time": null, - "max_height_cm": null, - "max_platforms_spot": null, - "meter_mark_cm": null, - "meter_mark_offset": null, - "model_name": null, - "one_d_sequence": null, - "paired_berth_gkey": null, - "parent_bin_gkey": { - "long": 1372043965 - }, - "pixels_between": null, - "pixels_stack_length": null, - "pixels_stack_width": null, - "prestage_group_gkey": null, - "purpose": null, - "road_access_delay_secs": null, - "roadway_column": null, - "rotate_90": null, - "row_index": null, - "row_paired_from": null, - "row_paired_into": null, - "rtg_lane_gkey": null, - "rtg_lane_name": null, - "saic_pocket_id": null, - "section_length_cm": null, - "ship_bay_gkey": null, - "show_conflicts": null, - "sort_logical": null, - "spots_are_relative": null, - "spots_show_r2l": null, - "sub_spot_name_table_gkey": null, - "sub_spots_r2l": null, - "sub_yard_code": null, - "table_offset": null, - "tcg_mode_enum": null, - "teu_capacity": null, - "vertical_max": null, - "vertical_min": null, - "yard_gkey": null, - "z_index_first": { - "long": 1 - }, - "z_index_last": { - "long": 5 - } - } - }, - "source": { - "version": "1.6.0-SNAPSHOT", - "connector": "oracle", - "name": "tos", - "ts_ms": 1667592158629, - "snapshot": { - "string": "true" - }, - "db": "NN4DSMT_AVLINO", - "sequence": null, - "schema": "N4_OWN", - "table": "SPATIAL_BINS_MV", - "txId": null, - "scn": { - "string": "1052993665" - }, - "commit_scn": null, - "lcr_position": null - }, - "op": "r", - "ts_ms": { - "long": 1667592158630 - }, - "transaction": null -} diff --git a/avro101/src/main/resources/al/data5_unitmoves_key_121.json b/avro101/src/main/resources/al/data5_unitmoves_key_121.json deleted file mode 100644 index 5cbfde20..00000000 --- a/avro101/src/main/resources/al/data5_unitmoves_key_121.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "container_terminal_visit_key": "636058214", - "move_event_key": "636077341" -} \ No newline at end of file diff --git a/avro101/src/main/resources/al/data5_unitmoves_value_122.json b/avro101/src/main/resources/al/data5_unitmoves_value_122.json deleted file mode 100644 index b3d1330f..00000000 --- a/avro101/src/main/resources/al/data5_unitmoves_value_122.json +++ /dev/null @@ -1,255 +0,0 @@ -{ - "moveEventKey": "636077341", - "containerTerminalVisitKey": "636058214", - "sequenceNumber": { - "long": 636077341 - }, - "moveKind": "RECV", - "containerCategory": "STRGE", - "containerFreightType": "MTY", - "isRehandle": false, - "isHousekeeping": false, - "pointOfWork": { - "string": "GATE" - }, - "rehandleCount": { - "int": 0 - }, - "lineOperatorTosKey": { - "string": "898183" - }, - "carrierTosKey": { - "string": "636058203" - }, - "dispatchTime": { - "long": 1597683640922 - }, - "fetchTime": { - "long": 0 - }, - "carryCompleteTime": { - "long": 1597683643594 - }, - "putTime": { - "long": 1597683643594 - }, - "fetchCheTosKey": null, - "fetchCheLoginName": null, - "isTwinFetch": { - "boolean": false - }, - "carryCheTosKey": null, - "carryCheLoginName": null, - "isTwinCarry": { - "boolean": false - }, - "putCheTosKey": { - "string": "63329921" - }, - "putCheLoginName": { - "string": "S874899" - }, - "isTwinPut": { - "boolean": false - }, - "qcCheTosKey": null, - "from": { - "com.avlino.alenza.aicon.events.movePart": { - "location": { - "com.avlino.alenza.aicon.entity.locationRecord": { - "locationType": "TRUCK", - "locationId": "2615730", - "locationTosKey": "636058203", - "slotPosition": "1", - "slotName": "T-2615730-1 (TIP)", - "stackPositionTier": -1, - "positionTosKey": "UNKNOWN", - "positionSubClass": "UNKNOWN", - "positionInfo": null - } - }, - "cheTosKey": { - "string": "63329921" - }, - "cheLoginName": { - "string": "S874899" - }, - "time": { - "long": 1597683643594 - } - } - }, - "to": { - "com.avlino.alenza.aicon.events.movePart": { - "location": { - "com.avlino.alenza.aicon.entity.locationRecord": { - "locationType": "YARD", - "locationId": "SMT", - "locationTosKey": "47076", - "slotPosition": "376M7.D", - "slotName": "Y-SMT-376M7D", - "stackPositionTier": 4, - "positionTosKey": "56552799", - "positionSubClass": "YST", - "positionInfo": { - "com.avlino.alenza.aicon.entity.PositionLevel": { - "level": "STACK", - "name": "376M7", - "size": { - "string": "FORTYFOOT" - }, - "parent": { - "com.avlino.alenza.aicon.entity.PositionLevel": { - "level": "BAY", - "name": "376", - "size": { - "string": "FORTYFOOT" - }, - "parent": { - "com.avlino.alenza.aicon.entity.PositionLevel": { - "level": "BLOCK", - "name": "300M", - "size": null, - "parent": { - "com.avlino.alenza.aicon.entity.PositionLevel": { - "level": "TERMINAL", - "name": "YARD", - "size": null, - "parent": { - "com.avlino.alenza.aicon.entity.PositionLevel": { - "level": "YARD", - "name": "MAIN", - "size": null, - "parent": null, - "head": null, - "tail": null - } - }, - "head": null, - "tail": null - } - }, - "head": null, - "tail": null - } - }, - "head": null, - "tail": null - } - }, - "head": { - "com.avlino.alenza.aicon.entity.PositionLevel": { - "level": "GROUNDSLOT", - "name": "375M7", - "size": null, - "parent": { - "com.avlino.alenza.aicon.entity.PositionLevel": { - "level": "BAY", - "name": "375", - "size": null, - "parent": { - "com.avlino.alenza.aicon.entity.PositionLevel": { - "level": "BLOCK", - "name": "300M", - "size": null, - "parent": { - "com.avlino.alenza.aicon.entity.PositionLevel": { - "level": "TERMINAL", - "name": "YARD", - "size": null, - "parent": { - "com.avlino.alenza.aicon.entity.PositionLevel": { - "level": "YARD", - "name": "MAIN", - "size": null, - "parent": null, - "head": null, - "tail": null - } - }, - "head": null, - "tail": null - } - }, - "head": null, - "tail": null - } - }, - "head": null, - "tail": null - } - }, - "head": null, - "tail": null - } - }, - "tail": { - "com.avlino.alenza.aicon.entity.PositionLevel": { - "level": "GROUNDSLOT", - "name": "377M7", - "size": null, - "parent": { - "com.avlino.alenza.aicon.entity.PositionLevel": { - "level": "BAY", - "name": "377", - "size": null, - "parent": { - "com.avlino.alenza.aicon.entity.PositionLevel": { - "level": "BLOCK", - "name": "300M", - "size": null, - "parent": { - "com.avlino.alenza.aicon.entity.PositionLevel": { - "level": "TERMINAL", - "name": "YARD", - "size": null, - "parent": { - "com.avlino.alenza.aicon.entity.PositionLevel": { - "level": "YARD", - "name": "MAIN", - "size": null, - "parent": null, - "head": null, - "tail": null - } - }, - "head": null, - "tail": null - } - }, - "head": null, - "tail": null - } - }, - "head": null, - "tail": null - } - }, - "head": null, - "tail": null - } - } - } - } - } - }, - "cheTosKey": { - "string": "63329921" - }, - "cheLoginName": { - "string": "S874899" - }, - "time": { - "long": 1597683643594 - } - } - }, - "isSnapshot": true, - "dbTime": { - "long": 0 - }, - "isDeleted": false, - "currentTimeMillis": { - "long": 1671121030627 - } -} \ No newline at end of file diff --git a/avro101/src/main/resources/al/kafka-topics.txt b/avro101/src/main/resources/al/kafka-topics.txt deleted file mode 100644 index b04085f7..00000000 --- a/avro101/src/main/resources/al/kafka-topics.txt +++ /dev/null @@ -1,4 +0,0 @@ -tos.N4_OWN.INV_MOVE_EVENT_MV (k:62,v:80) -tos.N4_OWN.SPATIAL_BINS_MV (k:57,v:65) -=> -aicon_events_unitmoves (k:121,v:122) diff --git a/avro101/src/main/resources/al/mag-map-syntrans-config-key.json b/avro101/src/main/resources/al/mag-map-syntrans-config-key.json deleted file mode 100644 index 8f97b262..00000000 --- a/avro101/src/main/resources/al/mag-map-syntrans-config-key.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "mappings" : [ - { - "to" : "container_terminal_visit_key", - "from" : "f1" - }, - { - "to" : "move_event_key", - "from" : "f2" - } - ] -} \ No newline at end of file diff --git a/avro101/src/main/resources/al/mag-map-syntrans-config-value.json b/avro101/src/main/resources/al/mag-map-syntrans-config-value.json deleted file mode 100644 index c7ef9dbe..00000000 --- a/avro101/src/main/resources/al/mag-map-syntrans-config-value.json +++ /dev/null @@ -1,132 +0,0 @@ -{ - "mappings" : [ - { - "to" : "moveKind", - "from" : "move_kind" - }, - { - "to" : "containerCategory", - "from" : "category" - }, - { - "to" : "containerFreightType", - "from" : "freight_kind" - }, - { - "to" : "pointOfWork", - "from" : "pow" - }, - { - "to" : "lineOperatorTosKey", - "from" : "line_op" - }, - { - "to" : "carrierTosKey", - "from" : "carrier_gkey" - }, - { - "to" : "fetchCheTosKey", - "from" : "che_fetch" - }, - { - "to" : "fetchCheLoginName", - "from" : "che_fetch_login_name" - }, - { - "to" : "carryCheTosKey", - "from" : "che_carry" - }, - { - "to" : "carryCheLoginName", - "from" : "che_carry_login_name" - }, - { - "to" : "putCheTosKey", - "from" : "che_put" - }, - { - "to" : "putCheLoginName", - "from" : "che_put_login_name" - }, - { - "to" : "qcCheTosKey", - "from" : "che_qc" - }, - { - "to" : "rehandleCount", - "from" : "rehandle_count" - }, - { - "to" : "sequenceNumber", - "from" : "mve_gkey" - }, - { - "to" : "dispatchTime", - "from" : "t_dispatch" - }, - { - "to" : "fetchTime", - "from" : "t_fetch" - }, - { - "to" : "carryCompleteTime", - "from" : "t_carry_complete" - }, - { - "to" : "putTime", - "from" : "t_put" - }, - { - "to" : "isTwinFetch", - "from" : "twin_fetch" - }, - { - "to" : "isTwinCarry", - "from" : "twin_carry" - }, - { - "to" : "isTwinPut", - "from" : "twin_put" - }, - { - "to" : "moveEventKey", - "from" : "f23_____" - }, - { - "to" : "containerTerminalVisitKey", - "from" : "f24_____" - }, - { - "to" : "isSnapshot", - "from" : "f25_____" - }, - { - "to" : "dbTime", - "from" : "f26_____" - }, - { - "to" : "isDeleted", - "from" : "f27_____" - }, - { - "to" : "from", - "from" : "f28_____" - }, - { - "to" : "to", - "from" : "f29_____" - }, - { - "to" : "isRehandle", - "from" : "f30_____" - }, - { - "to" : "isHousekeeping", - "from" : "f31_____" - }, - { - "to" : "currentTimeMillis", - "from" : "f32_____" - } - ] -} \ No newline at end of file diff --git a/avro101/src/main/resources/al/schema1_events_key_62.json b/avro101/src/main/resources/al/schema1_events_key_62.json deleted file mode 100644 index dcabe338..00000000 --- a/avro101/src/main/resources/al/schema1_events_key_62.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "type": "record", - "name": "Key", - "namespace": "tos.N4_OWN.INV_MOVE_EVENT_MV", - "fields": [ - { - "name": "mve_gkey", - "type": "long" - } - ], - "connect.name": "tos.N4_OWN.INV_MOVE_EVENT_MV.Key" -} diff --git a/avro101/src/main/resources/al/schema2_events_value_80.json b/avro101/src/main/resources/al/schema2_events_value_80.json deleted file mode 100644 index 95ea90b5..00000000 --- a/avro101/src/main/resources/al/schema2_events_value_80.json +++ /dev/null @@ -1,615 +0,0 @@ -{ - "type": "record", - "name": "Envelope", - "namespace": "tos.N4_OWN.INV_MOVE_EVENT_MV", - "fields": [ - { - "name": "before", - "type": [ - "null", - { - "type": "record", - "name": "Value", - "fields": [ - { - "name": "berth", - "type": [ - "null", - "string" - ], - "default": null - }, - { - "name": "carrier_gkey", - "type": [ - "null", - "long" - ], - "default": null - }, - { - "name": "category", - "type": [ - "null", - "string" - ], - "default": null - }, - { - "name": "che_carry", - "type": [ - "null", - "long" - ], - "default": null - }, - { - "name": "che_carry_login_name", - "type": [ - "null", - "string" - ], - "default": null - }, - { - "name": "che_fetch", - "type": [ - "null", - "long" - ], - "default": null - }, - { - "name": "che_fetch_login_name", - "type": [ - "null", - "string" - ], - "default": null - }, - { - "name": "che_put", - "type": [ - "null", - "long" - ], - "default": null - }, - { - "name": "che_put_login_name", - "type": [ - "null", - "string" - ], - "default": null - }, - { - "name": "che_qc", - "type": [ - "null", - "long" - ], - "default": null - }, - { - "name": "dist_carry", - "type": [ - "null", - "long" - ], - "default": null - }, - { - "name": "dist_start", - "type": [ - "null", - "long" - ], - "default": null - }, - { - "name": "exclude", - "type": [ - "null", - "boolean" - ], - "default": null - }, - { - "name": "fm_pos_bin", - "type": [ - "null", - "long" - ], - "default": null - }, - { - "name": "fm_pos_loc_gkey", - "type": [ - "null", - "long" - ], - "default": null - }, - { - "name": "fm_pos_locid", - "type": [ - "null", - "string" - ], - "default": null - }, - { - "name": "fm_pos_loctype", - "type": "string" - }, - { - "name": "fm_pos_name", - "type": [ - "null", - "string" - ], - "default": null - }, - { - "name": "fm_pos_orientation", - "type": [ - "null", - "string" - ], - "default": null - }, - { - "name": "fm_pos_orientation_degrees", - "type": [ - "null", - "double" - ], - "default": null - }, - { - "name": "fm_pos_slot", - "type": [ - "null", - "string" - ], - "default": null - }, - { - "name": "fm_pos_tier", - "type": [ - "null", - "long" - ], - "default": null - }, - { - "name": "freight_kind", - "type": [ - "null", - "string" - ], - "default": null - }, - { - "name": "line_op", - "type": [ - "null", - "long" - ], - "default": null - }, - { - "name": "move_kind", - "type": [ - "null", - "string" - ], - "default": null - }, - { - "name": "mve_gkey", - "type": "long" - }, - { - "name": "pow", - "type": [ - "null", - "string" - ], - "default": null - }, - { - "name": "processed", - "type": [ - "null", - "boolean" - ], - "default": null - }, - { - "name": "rehandle_count", - "type": [ - "null", - "long" - ], - "default": null - }, - { - "name": "restow_account", - "type": [ - "null", - "string" - ], - "default": null - }, - { - "name": "restow_reason", - "type": [ - "null", - "string" - ], - "default": null - }, - { - "name": "service_order", - "type": [ - "null", - "string" - ], - "default": null - }, - { - "name": "to_pos_bin", - "type": [ - "null", - "long" - ], - "default": null - }, - { - "name": "to_pos_locid", - "type": [ - "null", - "string" - ], - "default": null - }, - { - "name": "to_pos_loctype", - "type": "string" - }, - { - "name": "to_pos_loc_gkey", - "type": [ - "null", - "long" - ], - "default": null - }, - { - "name": "to_pos_name", - "type": [ - "null", - "string" - ], - "default": null - }, - { - "name": "to_pos_orientation", - "type": [ - "null", - "string" - ], - "default": null - }, - { - "name": "to_pos_orientation_degrees", - "type": [ - "null", - "double" - ], - "default": null - }, - { - "name": "to_pos_slot", - "type": [ - "null", - "string" - ], - "default": null - }, - { - "name": "to_pos_tier", - "type": [ - "null", - "long" - ], - "default": null - }, - { - "name": "twin_carry", - "type": [ - "null", - "boolean" - ], - "default": null - }, - { - "name": "twin_fetch", - "type": [ - "null", - "boolean" - ], - "default": null - }, - { - "name": "twin_put", - "type": [ - "null", - "boolean" - ], - "default": null - }, - { - "name": "t_carry_complete", - "type": [ - "null", - { - "type": "long", - "connect.version": 1, - "connect.name": "io.debezium.time.MicroTimestamp" - } - ], - "default": null - }, - { - "name": "t_carry_dispatch", - "type": [ - "null", - { - "type": "long", - "connect.version": 1, - "connect.name": "io.debezium.time.MicroTimestamp" - } - ], - "default": null - }, - { - "name": "t_carry_fetch_ready", - "type": [ - "null", - { - "type": "long", - "connect.version": 1, - "connect.name": "io.debezium.time.MicroTimestamp" - } - ], - "default": null - }, - { - "name": "t_carry_put_ready", - "type": [ - "null", - { - "type": "long", - "connect.version": 1, - "connect.name": "io.debezium.time.MicroTimestamp" - } - ], - "default": null - }, - { - "name": "t_discharge", - "type": [ - "null", - { - "type": "long", - "connect.version": 1, - "connect.name": "io.debezium.time.MicroTimestamp" - } - ], - "default": null - }, - { - "name": "t_dispatch", - "type": [ - "null", - { - "type": "long", - "connect.version": 1, - "connect.name": "io.debezium.time.MicroTimestamp" - } - ], - "default": null - }, - { - "name": "t_fetch", - "type": [ - "null", - { - "type": "long", - "connect.version": 1, - "connect.name": "io.debezium.time.MicroTimestamp" - } - ], - "default": null - }, - { - "name": "t_put", - "type": [ - "null", - { - "type": "long", - "connect.version": 1, - "connect.name": "io.debezium.time.MicroTimestamp" - } - ], - "default": null - }, - { - "name": "t_tz_arrival", - "type": [ - "null", - { - "type": "long", - "connect.version": 1, - "connect.name": "io.debezium.time.MicroTimestamp" - } - ], - "default": null - }, - { - "name": "ufv_gkey", - "type": "long" - } - ], - "connect.name": "tos.N4_OWN.INV_MOVE_EVENT_MV.Value" - } - ], - "default": null - }, - { - "name": "after", - "type": [ - "null", - "Value" - ], - "default": null - }, - { - "name": "source", - "type": { - "type": "record", - "name": "Source", - "namespace": "io.debezium.connector.oracle", - "fields": [ - { - "name": "version", - "type": "string" - }, - { - "name": "connector", - "type": "string" - }, - { - "name": "name", - "type": "string" - }, - { - "name": "ts_ms", - "type": "long" - }, - { - "name": "snapshot", - "type": [ - { - "type": "string", - "connect.version": 1, - "connect.parameters": { - "allowed": "true,last,false" - }, - "connect.default": "false", - "connect.name": "io.debezium.data.Enum" - }, - "null" - ], - "default": "false" - }, - { - "name": "db", - "type": "string" - }, - { - "name": "sequence", - "type": [ - "null", - "string" - ], - "default": null - }, - { - "name": "schema", - "type": "string" - }, - { - "name": "table", - "type": "string" - }, - { - "name": "txId", - "type": [ - "null", - "string" - ], - "default": null - }, - { - "name": "scn", - "type": [ - "null", - "string" - ], - "default": null - }, - { - "name": "commit_scn", - "type": [ - "null", - "string" - ], - "default": null - }, - { - "name": "lcr_position", - "type": [ - "null", - "string" - ], - "default": null - } - ], - "connect.name": "io.debezium.connector.oracle.Source" - } - }, - { - "name": "op", - "type": "string" - }, - { - "name": "ts_ms", - "type": [ - "null", - "long" - ], - "default": null - }, - { - "name": "transaction", - "type": [ - "null", - { - "type": "record", - "name": "ConnectDefault", - "namespace": "io.confluent.connect.avro", - "fields": [ - { - "name": "id", - "type": "string" - }, - { - "name": "total_order", - "type": "long" - }, - { - "name": "data_collection_order", - "type": "long" - } - ] - } - ], - "default": null - } - ], - "connect.name": "tos.N4_OWN.INV_MOVE_EVENT_MV.Envelope" -} diff --git a/avro101/src/main/resources/al/schema3_bins_key_57.json b/avro101/src/main/resources/al/schema3_bins_key_57.json deleted file mode 100644 index 2aa81178..00000000 --- a/avro101/src/main/resources/al/schema3_bins_key_57.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "type": "record", - "name": "Key", - "namespace": "tos.N4_OWN.SPATIAL_BINS_MV", - "fields": [ - { - "name": "gkey", - "type": "long" - } - ], - "connect.name": "tos.N4_OWN.SPATIAL_BINS_MV.Key" -} diff --git a/avro101/src/main/resources/al/schema4_bins_value_65.json b/avro101/src/main/resources/al/schema4_bins_value_65.json deleted file mode 100644 index e171520b..00000000 --- a/avro101/src/main/resources/al/schema4_bins_value_65.json +++ /dev/null @@ -1,1139 +0,0 @@ -{ - "type": "record", - "name": "Envelope", - "namespace": "tos.N4_OWN.SPATIAL_BINS_MV", - "fields": [ - { - "name": "before", - "type": [ - "null", - { - "type": "record", - "name": "Value", - "fields": [ - { - "name": "adjacent_block_gkey", - "type": [ - "null", - "long" - ], - "default": null - }, - { - "name": "allows_20s", - "type": [ - "null", - "boolean" - ], - "default": null - }, - { - "name": "allows_24s", - "type": [ - "null", - "boolean" - ], - "default": null - }, - { - "name": "allows_30s", - "type": [ - "null", - "boolean" - ], - "default": null - }, - { - "name": "allows_35s", - "type": [ - "null", - "boolean" - ], - "default": null - }, - { - "name": "allows_40s", - "type": [ - "null", - "boolean" - ], - "default": null - }, - { - "name": "allows_45s", - "type": [ - "null", - "boolean" - ], - "default": null - }, - { - "name": "allows_48s", - "type": [ - "null", - "boolean" - ], - "default": null - }, - { - "name": "allows_53s", - "type": [ - "null", - "boolean" - ], - "default": null - }, - { - "name": "anchor_geodetic_orientation", - "type": [ - "null", - "double" - ], - "default": null - }, - { - "name": "anchor_latitude", - "type": [ - "null", - "double" - ], - "default": null - }, - { - "name": "anchor_longitude", - "type": [ - "null", - "double" - ], - "default": null - }, - { - "name": "angle_rotation", - "type": [ - "null", - "double" - ], - "default": null - }, - { - "name": "bin_level", - "type": [ - "null", - "long" - ], - "default": null - }, - { - "name": "bin_subclass", - "type": "string" - }, - { - "name": "bin_type", - "type": [ - "null", - "long" - ], - "default": null - }, - { - "name": "block_group", - "type": [ - "null", - "string" - ], - "default": null - }, - { - "name": "block_type_enum", - "type": [ - "null", - "string" - ], - "default": null - }, - { - "name": "changed", - "type": [ - "null", - { - "type": "long", - "connect.version": 1, - "connect.name": "io.debezium.time.MicroTimestamp" - } - ], - "default": null - }, - { - "name": "changer", - "type": [ - "null", - "string" - ], - "default": null - }, - { - "name": "che_kind_1", - "type": [ - "null", - "string" - ], - "default": null - }, - { - "name": "che_kind_2", - "type": [ - "null", - "string" - ], - "default": null - }, - { - "name": "code_chars", - "type": [ - "null", - "string" - ], - "default": null - }, - { - "name": "column_index", - "type": [ - "null", - "long" - ], - "default": null - }, - { - "name": "compress_scan", - "type": [ - "null", - "boolean" - ], - "default": null - }, - { - "name": "conv_gkey_row_alt", - "type": [ - "null", - "long" - ], - "default": null - }, - { - "name": "conv_gkey_row_std", - "type": [ - "null", - "long" - ], - "default": null - }, - { - "name": "conv_gkey_tier", - "type": [ - "null", - "long" - ], - "default": null - }, - { - "name": "conv_gkey_tier_csn", - "type": [ - "null", - "long" - ], - "default": null - }, - { - "name": "coord_col_conv_id", - "type": [ - "null", - "long" - ], - "default": null - }, - { - "name": "coord_col_conv_id_40", - "type": [ - "null", - "long" - ], - "default": null - }, - { - "name": "created", - "type": [ - "null", - { - "type": "long", - "connect.version": 1, - "connect.name": "io.debezium.time.MicroTimestamp" - } - ], - "default": null - }, - { - "name": "creator", - "type": [ - "null", - "string" - ], - "default": null - }, - { - "name": "default_col_spacing_cm", - "type": [ - "null", - "double" - ], - "default": null - }, - { - "name": "default_col_width_cm", - "type": [ - "null", - "double" - ], - "default": null - }, - { - "name": "default_doors", - "type": [ - "null", - "string" - ], - "default": null - }, - { - "name": "default_row_spacing_cm", - "type": [ - "null", - "double" - ], - "default": null - }, - { - "name": "default_row_width_cm", - "type": [ - "null", - "double" - ], - "default": null - }, - { - "name": "direction", - "type": [ - "null", - "string" - ], - "default": null - }, - { - "name": "display_order_index", - "type": [ - "null", - "long" - ], - "default": null - }, - { - "name": "dynamic_traffic_enum", - "type": [ - "null", - "string" - ], - "default": null - }, - { - "name": "extend_x_20s", - "type": [ - "null", - "boolean" - ], - "default": null - }, - { - "name": "extend_x_24s", - "type": [ - "null", - "boolean" - ], - "default": null - }, - { - "name": "extend_x_30s", - "type": [ - "null", - "boolean" - ], - "default": null - }, - { - "name": "extend_x_35s", - "type": [ - "null", - "boolean" - ], - "default": null - }, - { - "name": "extend_x_40s", - "type": [ - "null", - "boolean" - ], - "default": null - }, - { - "name": "extend_x_45s", - "type": [ - "null", - "boolean" - ], - "default": null - }, - { - "name": "extend_x_48s", - "type": [ - "null", - "boolean" - ], - "default": null - }, - { - "name": "extend_x_53s", - "type": [ - "null", - "boolean" - ], - "default": null - }, - { - "name": "facing_direction_deg", - "type": [ - "null", - "double" - ], - "default": null - }, - { - "name": "forties_extend_ascending", - "type": [ - "null", - "boolean" - ], - "default": null - }, - { - "name": "geometry_rotation", - "type": [ - "null", - "double" - ], - "default": null - }, - { - "name": "gkey", - "type": "long" - }, - { - "name": "grid_extends_east", - "type": [ - "null", - "boolean" - ], - "default": null - }, - { - "name": "grid_extends_north", - "type": [ - "null", - "boolean" - ], - "default": null - }, - { - "name": "grid_rotation", - "type": [ - "null", - "double" - ], - "default": null - }, - { - "name": "groove", - "type": [ - "null", - "long" - ], - "default": null - }, - { - "name": "has_centerline_stack", - "type": [ - "null", - "boolean" - ], - "default": null - }, - { - "name": "icon_id", - "type": [ - "null", - "long" - ], - "default": null - }, - { - "name": "index_within_track", - "type": [ - "null", - "long" - ], - "default": null - }, - { - "name": "internal_pos_name", - "type": [ - "null", - "string" - ], - "default": null - }, - { - "name": "internal_pos_name_alt", - "type": [ - "null", - "string" - ], - "default": null - }, - { - "name": "invert_scan_x", - "type": [ - "null", - "boolean" - ], - "default": null - }, - { - "name": "invert_scan_y", - "type": [ - "null", - "boolean" - ], - "default": null - }, - { - "name": "invert_x", - "type": [ - "null", - "boolean" - ], - "default": null - }, - { - "name": "invert_y", - "type": [ - "null", - "boolean" - ], - "default": null - }, - { - "name": "is_access_blocked_first_col", - "type": [ - "null", - "boolean" - ], - "default": null - }, - { - "name": "is_access_blocked_last_col", - "type": [ - "null", - "boolean" - ], - "default": null - }, - { - "name": "label_scheme_col_only", - "type": [ - "null", - "string" - ], - "default": null - }, - { - "name": "label_scheme_full_position", - "type": [ - "null", - "string" - ], - "default": null - }, - { - "name": "label_scheme_host", - "type": [ - "null", - "string" - ], - "default": null - }, - { - "name": "label_scheme_row_only", - "type": [ - "null", - "string" - ], - "default": null - }, - { - "name": "label_scheme_tier_only", - "type": [ - "null", - "string" - ], - "default": null - }, - { - "name": "lcg_mode_enum", - "type": [ - "null", - "string" - ], - "default": null - }, - { - "name": "lcg_ref_enum", - "type": [ - "null", - "string" - ], - "default": null - }, - { - "name": "lengths_allowed", - "type": [ - "null", - "string" - ], - "default": null - }, - { - "name": "level_logical_index", - "type": [ - "null", - "long" - ], - "default": null - }, - { - "name": "lid_index_first", - "type": [ - "null", - "long" - ], - "default": null - }, - { - "name": "lid_index_last", - "type": [ - "null", - "long" - ], - "default": null - }, - { - "name": "life_cycle_state", - "type": "string" - }, - { - "name": "long_name", - "type": [ - "null", - "string" - ], - "default": null - }, - { - "name": "map_image", - "type": [ - "null", - "bytes" - ], - "default": null - }, - { - "name": "map_image_update_time", - "type": [ - "null", - { - "type": "long", - "connect.version": 1, - "connect.name": "io.debezium.time.MicroTimestamp" - } - ], - "default": null - }, - { - "name": "max_height_cm", - "type": [ - "null", - "double" - ], - "default": null - }, - { - "name": "max_platforms_spot", - "type": [ - "null", - "long" - ], - "default": null - }, - { - "name": "meter_mark_cm", - "type": [ - "null", - "double" - ], - "default": null - }, - { - "name": "meter_mark_offset", - "type": [ - "null", - "double" - ], - "default": null - }, - { - "name": "model_name", - "type": [ - "null", - "string" - ], - "default": null - }, - { - "name": "one_d_sequence", - "type": [ - "null", - "long" - ], - "default": null - }, - { - "name": "paired_berth_gkey", - "type": [ - "null", - "long" - ], - "default": null - }, - { - "name": "parent_bin_gkey", - "type": [ - "null", - "long" - ], - "default": null - }, - { - "name": "pixels_between", - "type": [ - "null", - "long" - ], - "default": null - }, - { - "name": "pixels_stack_length", - "type": [ - "null", - "long" - ], - "default": null - }, - { - "name": "pixels_stack_width", - "type": [ - "null", - "long" - ], - "default": null - }, - { - "name": "prestage_group_gkey", - "type": [ - "null", - "long" - ], - "default": null - }, - { - "name": "purpose", - "type": [ - "null", - "string" - ], - "default": null - }, - { - "name": "road_access_delay_secs", - "type": [ - "null", - "long" - ], - "default": null - }, - { - "name": "roadway_column", - "type": [ - "null", - "long" - ], - "default": null - }, - { - "name": "rotate_90", - "type": [ - "null", - "boolean" - ], - "default": null - }, - { - "name": "row_index", - "type": [ - "null", - "long" - ], - "default": null - }, - { - "name": "row_paired_from", - "type": [ - "null", - "long" - ], - "default": null - }, - { - "name": "row_paired_into", - "type": [ - "null", - "long" - ], - "default": null - }, - { - "name": "rtg_lane_gkey", - "type": [ - "null", - "long" - ], - "default": null - }, - { - "name": "rtg_lane_name", - "type": [ - "null", - "string" - ], - "default": null - }, - { - "name": "saic_pocket_id", - "type": [ - "null", - "long" - ], - "default": null - }, - { - "name": "section_length_cm", - "type": [ - "null", - "double" - ], - "default": null - }, - { - "name": "ship_bay_gkey", - "type": [ - "null", - "long" - ], - "default": null - }, - { - "name": "show_conflicts", - "type": [ - "null", - "boolean" - ], - "default": null - }, - { - "name": "sort_logical", - "type": [ - "null", - "boolean" - ], - "default": null - }, - { - "name": "spots_are_relative", - "type": [ - "null", - "boolean" - ], - "default": null - }, - { - "name": "spots_show_r2l", - "type": [ - "null", - "boolean" - ], - "default": null - }, - { - "name": "sub_spot_name_table_gkey", - "type": [ - "null", - "long" - ], - "default": null - }, - { - "name": "sub_spots_r2l", - "type": [ - "null", - "boolean" - ], - "default": null - }, - { - "name": "sub_yard_code", - "type": [ - "null", - "string" - ], - "default": null - }, - { - "name": "table_offset", - "type": [ - "null", - "long" - ], - "default": null - }, - { - "name": "tcg_mode_enum", - "type": [ - "null", - "string" - ], - "default": null - }, - { - "name": "teu_capacity", - "type": [ - "null", - "long" - ], - "default": null - }, - { - "name": "vertical_max", - "type": [ - "null", - "double" - ], - "default": null - }, - { - "name": "vertical_min", - "type": [ - "null", - "double" - ], - "default": null - }, - { - "name": "yard_gkey", - "type": [ - "null", - "long" - ], - "default": null - }, - { - "name": "z_index_first", - "type": [ - "null", - "long" - ], - "default": null - }, - { - "name": "z_index_last", - "type": [ - "null", - "long" - ], - "default": null - } - ], - "connect.name": "tos.N4_OWN.SPATIAL_BINS_MV.Value" - } - ], - "default": null - }, - { - "name": "after", - "type": [ - "null", - "Value" - ], - "default": null - }, - { - "name": "source", - "type": { - "type": "record", - "name": "Source", - "namespace": "io.debezium.connector.oracle", - "fields": [ - { - "name": "version", - "type": "string" - }, - { - "name": "connector", - "type": "string" - }, - { - "name": "name", - "type": "string" - }, - { - "name": "ts_ms", - "type": "long" - }, - { - "name": "snapshot", - "type": [ - { - "type": "string", - "connect.version": 1, - "connect.parameters": { - "allowed": "true,last,false" - }, - "connect.default": "false", - "connect.name": "io.debezium.data.Enum" - }, - "null" - ], - "default": "false" - }, - { - "name": "db", - "type": "string" - }, - { - "name": "sequence", - "type": [ - "null", - "string" - ], - "default": null - }, - { - "name": "schema", - "type": "string" - }, - { - "name": "table", - "type": "string" - }, - { - "name": "txId", - "type": [ - "null", - "string" - ], - "default": null - }, - { - "name": "scn", - "type": [ - "null", - "string" - ], - "default": null - }, - { - "name": "commit_scn", - "type": [ - "null", - "string" - ], - "default": null - }, - { - "name": "lcr_position", - "type": [ - "null", - "string" - ], - "default": null - } - ], - "connect.name": "io.debezium.connector.oracle.Source" - } - }, - { - "name": "op", - "type": "string" - }, - { - "name": "ts_ms", - "type": [ - "null", - "long" - ], - "default": null - }, - { - "name": "transaction", - "type": [ - "null", - { - "type": "record", - "name": "ConnectDefault", - "namespace": "io.confluent.connect.avro", - "fields": [ - { - "name": "id", - "type": "string" - }, - { - "name": "total_order", - "type": "long" - }, - { - "name": "data_collection_order", - "type": "long" - } - ] - } - ], - "default": null - } - ], - "connect.name": "tos.N4_OWN.SPATIAL_BINS_MV.Envelope" -} diff --git a/avro101/src/main/resources/al/schema5_unitmoves_key_121.json b/avro101/src/main/resources/al/schema5_unitmoves_key_121.json deleted file mode 100644 index 4ab0ac92..00000000 --- a/avro101/src/main/resources/al/schema5_unitmoves_key_121.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "type": "record", - "name": "UnitMovesKey", - "namespace": "com.avlino.alenza.aicon.events", - "fields": [ - { - "name": "container_terminal_visit_key", - "type": "string" - }, - { - "name": "move_event_key", - "type": "string" - } - ] -} diff --git a/avro101/src/main/resources/al/schema6_unitmoves_value_122.json b/avro101/src/main/resources/al/schema6_unitmoves_value_122.json deleted file mode 100644 index 80d5cea4..00000000 --- a/avro101/src/main/resources/al/schema6_unitmoves_value_122.json +++ /dev/null @@ -1,401 +0,0 @@ -{ - "type": "record", - "name": "UnitMovesValue", - "namespace": "com.avlino.alenza.aicon.events", - "fields": [ - { - "name": "moveEventKey", - "type": "string", - "default": "UNKNOWN" - }, - { - "name": "containerTerminalVisitKey", - "type": "string", - "default": "UNKNOWN" - }, - { - "name": "sequenceNumber", - "type": [ - "null", - "long" - ], - "default": null - }, - { - "name": "moveKind", - "type": "string", - "default": "UNKNOWN" - }, - { - "name": "containerCategory", - "type": "string", - "default": "UNKNOWN" - }, - { - "name": "containerFreightType", - "type": "string", - "default": "UNKNOWN" - }, - { - "name": "isRehandle", - "type": "boolean", - "default": false - }, - { - "name": "isHousekeeping", - "type": "boolean", - "default": false - }, - { - "name": "pointOfWork", - "type": [ - "null", - "string" - ], - "default": null - }, - { - "name": "rehandleCount", - "type": [ - "null", - "int" - ], - "default": null - }, - { - "name": "lineOperatorTosKey", - "type": [ - "null", - "string" - ], - "default": null - }, - { - "name": "carrierTosKey", - "type": [ - "null", - "string" - ], - "default": null - }, - { - "name": "dispatchTime", - "type": [ - "null", - { - "type": "long", - "connect.version": 1, - "connect.name": "org.apache.kafka.connect.data.Timestamp", - "logicalType": "timestamp-millis" - } - ], - "default": null - }, - { - "name": "fetchTime", - "type": [ - "null", - { - "type": "long", - "connect.version": 1, - "connect.name": "org.apache.kafka.connect.data.Timestamp", - "logicalType": "timestamp-millis" - } - ], - "default": null - }, - { - "name": "carryCompleteTime", - "type": [ - "null", - { - "type": "long", - "connect.version": 1, - "connect.name": "org.apache.kafka.connect.data.Timestamp", - "logicalType": "timestamp-millis" - } - ], - "default": null - }, - { - "name": "putTime", - "type": [ - "null", - { - "type": "long", - "connect.version": 1, - "connect.name": "org.apache.kafka.connect.data.Timestamp", - "logicalType": "timestamp-millis" - } - ], - "default": null - }, - { - "name": "fetchCheTosKey", - "type": [ - "null", - "string" - ], - "default": null - }, - { - "name": "fetchCheLoginName", - "type": [ - "null", - "string" - ], - "default": null - }, - { - "name": "isTwinFetch", - "type": [ - "null", - "boolean" - ], - "default": null - }, - { - "name": "carryCheTosKey", - "type": [ - "null", - "string" - ], - "default": null - }, - { - "name": "carryCheLoginName", - "type": [ - "null", - "string" - ], - "default": null - }, - { - "name": "isTwinCarry", - "type": [ - "null", - "boolean" - ], - "default": null - }, - { - "name": "putCheTosKey", - "type": [ - "null", - "string" - ], - "default": null - }, - { - "name": "putCheLoginName", - "type": [ - "null", - "string" - ], - "default": null - }, - { - "name": "isTwinPut", - "type": [ - "null", - "boolean" - ], - "default": null - }, - { - "name": "qcCheTosKey", - "type": [ - "null", - "string" - ], - "default": null - }, - { - "name": "from", - "type": [ - "null", - { - "type": "record", - "name": "movePart", - "fields": [ - { - "name": "location", - "type": [ - "null", - { - "type": "record", - "name": "locationRecord", - "namespace": "com.avlino.alenza.aicon.entity", - "fields": [ - { - "name": "locationType", - "type": "string" - }, - { - "name": "locationId", - "type": "string" - }, - { - "name": "locationTosKey", - "type": "string" - }, - { - "name": "slotPosition", - "type": "string" - }, - { - "name": "slotName", - "type": "string", - "default": "UNKNOWN" - }, - { - "name": "stackPositionTier", - "type": "int", - "default": -1 - }, - { - "name": "positionTosKey", - "type": "string", - "default": "UNKNOWN" - }, - { - "name": "positionSubClass", - "type": "string", - "default": "UNKNOWN" - }, - { - "name": "positionInfo", - "type": [ - "null", - { - "type": "record", - "name": "PositionLevel", - "fields": [ - { - "name": "level", - "type": "string" - }, - { - "name": "name", - "type": "string" - }, - { - "name": "size", - "type": [ - "null", - "string" - ], - "default": null - }, - { - "name": "parent", - "type": [ - "null", - "PositionLevel" - ], - "default": null - }, - { - "name": "head", - "type": [ - "null", - "PositionLevel" - ], - "default": null - }, - { - "name": "tail", - "type": [ - "null", - "PositionLevel" - ], - "default": null - } - ] - } - ] - } - ] - } - ] - }, - { - "name": "cheTosKey", - "type": [ - "null", - "string" - ], - "default": null - }, - { - "name": "cheLoginName", - "type": [ - "null", - "string" - ], - "default": null - }, - { - "name": "time", - "type": [ - "null", - { - "type": "long", - "connect.version": 1, - "connect.name": "org.apache.kafka.connect.data.Timestamp", - "logicalType": "timestamp-millis" - } - ], - "default": null - } - ] - } - ], - "default": null - }, - { - "name": "to", - "type": [ - "null", - "movePart" - ], - "default": null - }, - { - "name": "isSnapshot", - "type": "boolean", - "default": true - }, - { - "name": "dbTime", - "type": [ - "null", - { - "type": "long", - "connect.version": 1, - "connect.name": "org.apache.kafka.connect.data.Timestamp", - "logicalType": "timestamp-millis" - } - ], - "default": null - }, - { - "name": "isDeleted", - "type": "boolean", - "default": false - }, - { - "name": "currentTimeMillis", - "type": [ - "null", - { - "type": "long", - "connect.version": 1, - "connect.name": "org.apache.kafka.connect.data.Timestamp", - "logicalType": "timestamp-millis" - } - ], - "default": null - } - ] -} \ No newline at end of file diff --git a/avro101/src/main/resources/static/0 b/avro101/src/main/resources/static/0 deleted file mode 100644 index 3c546eb2..00000000 --- a/avro101/src/main/resources/static/0 +++ /dev/null @@ -1 +0,0 @@ -dummy file \ No newline at end of file diff --git a/avro101/src/main/resources/static/1.txt b/avro101/src/main/resources/static/1.txt deleted file mode 100644 index 2df989af..00000000 --- a/avro101/src/main/resources/static/1.txt +++ /dev/null @@ -1 +0,0 @@ -real file #1 \ No newline at end of file diff --git a/avro101/src/main/resources/static/2.txt b/avro101/src/main/resources/static/2.txt deleted file mode 100644 index d7b5fc06..00000000 --- a/avro101/src/main/resources/static/2.txt +++ /dev/null @@ -1 +0,0 @@ -real file #2 \ No newline at end of file diff --git a/avro101/src/main/scala/Playground2.scala b/avro101/src/main/scala/Playground2.scala deleted file mode 100644 index 6ba406c5..00000000 --- a/avro101/src/main/scala/Playground2.scala +++ /dev/null @@ -1,34 +0,0 @@ -import org.apache.avro.generic.{GenericData, GenericRecord} -import pprint.{pprintln => println} -import tools.Console.delimiter - -object Playground2 extends App { - - import avlx.DataCreate._ - import genrec.syntax._ - - val x1: Option[String] = user1.go[String]("name") - val x2: Option[Int] = user1.go[Int]("age") - println(x1) - println(x2) - - delimiter() - - val p: GenericData.Record = user6 - - val f1: Option[String] = p.go[String]("name") - val f2: Option[GenericRecord] = p.go[GenericRecord]("address") - val f3: Option[String] = p.go[String]("address.street") - val f4: Option[Long] = p.go[Long]("address.house") - val f5: Option[Long] = p.go[Long]("a") - val f6: Option[Double] = p.go[Double]("address.street.x") - - println(p) - println(p.getSchema) - println(f1) - println(f2) - println(f3) - println(f4) - println(f5) - println(f6) -} diff --git a/avro101/src/main/scala/ResourcesPlayground.scala b/avro101/src/main/scala/ResourcesPlayground.scala deleted file mode 100644 index 35e1a747..00000000 --- a/avro101/src/main/scala/ResourcesPlayground.scala +++ /dev/null @@ -1,17 +0,0 @@ -import java.io.File -import java.net.URL - -object ResourcesPlayground extends App { - - val c: Class[_] = getClass - val cl: ClassLoader = c.getClassLoader - val r: URL = cl.getResource("static/0") - val f: String = r.getFile - val path = f.substring(0, f.length-1) - - val f1 = new File(path + "1.txt") - - println(f1) - println(f1.exists) - -} diff --git a/avro101/src/main/scala/avlx/DataCreate.scala b/avro101/src/main/scala/avlx/DataCreate.scala deleted file mode 100644 index 4b018bcb..00000000 --- a/avro101/src/main/scala/avlx/DataCreate.scala +++ /dev/null @@ -1,35 +0,0 @@ -package avlx - -import org.apache.avro.generic.GenericData - -object DataCreate { - - /** record create */ - val user1: GenericData.Record = new GenericData.Record(SchemaCreate.schema1a) - user1.put("name", "Jim") - user1.put("age", 33) - - val user2: GenericData.Record = new GenericData.Record(SchemaCreate.schema1a) - user2.put("name", "Jack") - user2.put("age", 44) - - val user3: GenericData.Record = new GenericData.Record(SchemaCreate.schema1a) - user3.put("name", "Alex") - - val user4: GenericData.Record = new GenericData.Record(SchemaCreate.schema1a) - user4.put("age", 55) - - val user5: GenericData.Record = new GenericData.Record(SchemaCreate.schema1a) - - /** generic data fails only on absent attribute names */ - // user1.put("whatever", new Object) - - val address = new GenericData.Record(SchemaCreate.addressSchema) - address.put("street", "Broadway") - address.put("house", 123L) - - val user6 = new GenericData.Record(SchemaCreate.schema1d) - user6.put("name", "Jackery") - user6.put("age", 66) - user6.put("address", address) -} diff --git a/avro101/src/main/scala/avlx/DataRead.scala b/avro101/src/main/scala/avlx/DataRead.scala deleted file mode 100644 index a784c0a7..00000000 --- a/avro101/src/main/scala/avlx/DataRead.scala +++ /dev/null @@ -1,30 +0,0 @@ -package avlx - -import tools.Console.delimiter -import tools.Tools.readFrom - -import java.io.File - -object DataRead { - - def main(as: Array[String]): Unit = { - val FILE = new File("persons.avro") - - /** read without schema enforced */ - readFrom(FILE) - .foreach { x => println(s"schema: ${x.getSchema}, data: $x") } - - delimiter() - - /** read WITH schema enforced */ - readFrom(FILE, SchemaCreate.schema1a) - .foreach { x => println(s"schema: ${x.getSchema}, data: $x") } - - delimiter() - - /** read WITH schema enforced, should fail, schema specified has one field isn't exist on th data saved */ - readFrom(FILE, SchemaCreate.schema1c) - .foreach { x => println(s"schema: ${x.getSchema}, data: $x") } - } - -} diff --git a/avro101/src/main/scala/avlx/DataValidate.scala b/avro101/src/main/scala/avlx/DataValidate.scala deleted file mode 100644 index 06a0fb0d..00000000 --- a/avro101/src/main/scala/avlx/DataValidate.scala +++ /dev/null @@ -1,25 +0,0 @@ -package avlx - -import avlx.DataCreate._ -import tools.Tools.validateRecord - -object DataValidate { - - def main(as: Array[String]): Unit = { - println(user1) // {"name": "Jim", "age": 33} - println(validateRecord(user1)) - - println(user2) // {"name": "Jack", "age": 44} - println(validateRecord(user2)) - - println(user3) // {"name": "Alex", "age": null} - println(validateRecord(user3)) - - println(user4) // {"name": null, "age": 55} - println(validateRecord(user4)) // false - - println(user5) // {"name": null, "age": null} - println(validateRecord(user5)) // false - } - -} diff --git a/avro101/src/main/scala/avlx/DataWrite.scala b/avro101/src/main/scala/avlx/DataWrite.scala deleted file mode 100644 index e1b20824..00000000 --- a/avro101/src/main/scala/avlx/DataWrite.scala +++ /dev/null @@ -1,16 +0,0 @@ -package avlx - -import tools.Tools.writeTo - -import java.io.File - -object DataWrite { - - import DataCreate._ - - def main(as: Array[String]): Unit = { - val FILE = new File("persons.avro") - writeTo(FILE, SchemaCreate.schema1a, user1, user2, user3) - } - -} diff --git a/avro101/src/main/scala/avlx/SchemaCreate.scala b/avro101/src/main/scala/avlx/SchemaCreate.scala deleted file mode 100644 index ced9d71d..00000000 --- a/avro101/src/main/scala/avlx/SchemaCreate.scala +++ /dev/null @@ -1,43 +0,0 @@ -package avlx - -import org.apache.avro.Schema -import org.apache.avro.SchemaBuilder -import tools.Utils.contentsFromResources - -object SchemaCreate { - - /** create schema programatically */ - // format: off - val schema1a = SchemaBuilder.record("person").fields() - .name("name").`type`.stringType.noDefault - .name("age").`type`.nullable.intType.noDefault - .endRecord - /** create the same schema by reading from file */ - val contents = contentsFromResources("0/schema1.json") - val schema1b = new Schema.Parser().parse(contents) - - val schema1c = SchemaBuilder.record("person").fields() - .name("name").`type`.stringType.noDefault - .name("age").`type`.nullable.intType.noDefault - .name("skill").`type`.stringType.noDefault - .endRecord - - val addressSchema = SchemaBuilder.record("address").fields() - .name("street").`type`.stringType.noDefault - .name("house").`type`.longType().noDefault - .endRecord() - - val schema1d = SchemaBuilder.record("person2").fields() - .name("name").`type`.stringType.noDefault - .name("age").`type`.nullable.intType.noDefault - .name("address").`type`(addressSchema).noDefault - .endRecord - // format: on - - def main(as: Array[String]) = { - println(schema1a) - println(schema1b) - println(schema1d) - } - -} diff --git a/avro101/src/main/scala/genrec/TypeExtractor.scala b/avro101/src/main/scala/genrec/TypeExtractor.scala deleted file mode 100644 index 0b11908b..00000000 --- a/avro101/src/main/scala/genrec/TypeExtractor.scala +++ /dev/null @@ -1,71 +0,0 @@ -package genrec - -import cats.implicits.catsSyntaxOptionId -import org.apache.avro.generic.GenericRecord - -trait TypeExtractor[A] { - def extract(x: Any): Option[A] -} - -object TypeExtractor { - - type Node = GenericRecord - - implicit val typeExtractorBool: TypeExtractor[Boolean] = { - case x: Boolean => x.some - case _ => None - } - - implicit val typeExtractorChar: TypeExtractor[Char] = { - case x: Char => x.some - case _ => None - } - - implicit val typeExtractorLong: TypeExtractor[Long] = { - case x: Long => x.some - case x: Int => x.toLong.some - case x: Short => x.toLong.some - case x: Byte => x.toLong.some - case _ => None - } - - implicit val typeExtractorInt: TypeExtractor[Int] = { - case x: Int => x.some - case x: Short => x.toInt.some - case x: Byte => x.toInt.some - case _ => None - } - - implicit val typeExtractorShort: TypeExtractor[Short] = { - case x: Short => x.some - case x: Byte => x.toShort.some - case _ => None - } - - implicit val typeExtractorByte: TypeExtractor[Byte] = { - case x: Byte => x.some - case _ => None - } - - implicit val typeExtractorFloat: TypeExtractor[Float] = { - case x: Float => x.some - case _ => None - } - - implicit val typeExtractorDouble: TypeExtractor[Double] = { - case x: Double => x.some - case x: Float => x.toDouble.some - case _ => None - } - - implicit val typeExtractorString: TypeExtractor[String] = { - case x: String => x.some - case _ => None - } - - implicit val typeExtractorNode: TypeExtractor[Node] = { - case x: Node => x.some - case _ => None - } - -} diff --git a/avro101/src/main/scala/genrec/syntax.scala b/avro101/src/main/scala/genrec/syntax.scala deleted file mode 100644 index 70789e11..00000000 --- a/avro101/src/main/scala/genrec/syntax.scala +++ /dev/null @@ -1,32 +0,0 @@ -package genrec - -import org.apache.avro.generic.GenericRecord -import scala.annotation.tailrec - -object syntax { - - implicit class GenericRecordOps(record: GenericRecord) { - - // TODO: since this is Avro an it has a schema - // we can think of detailed error support like: - // - field `name` doesn't exist in a given schema on the corresponding layer - // - field `name` exists but has a different type ... - def go[A](path: String)(implicit ex: TypeExtractor[A], gx: TypeExtractor[GenericRecord]): Option[A] = { - - @tailrec - def navigate(cursor: GenericRecord, path: List[String]): Option[A] = path match { - case Nil => None - case x :: Nil => ex.extract(cursor.get(x)) - case x :: t => - gx.extract(cursor.get(x)) match { - case Some(r) => navigate(r, t) - case None => None - } - } - - navigate(record, path.split("\\.").toList) - } - - } - -} diff --git a/avro101/src/main/scala/joiner/Joiner.scala b/avro101/src/main/scala/joiner/Joiner.scala deleted file mode 100644 index a319aec4..00000000 --- a/avro101/src/main/scala/joiner/Joiner.scala +++ /dev/null @@ -1,130 +0,0 @@ -package joiner - -import joiner.AgnosticJoiner.{HashMapJoiner, IterableJoiner, KMaterialized, KafkaStreamsJoiner} -import org.apache.kafka.streams.scala.ByteArrayKeyValueStore -import org.apache.kafka.streams.scala.kstream.KTable -import org.apache.kafka.streams.scala.kstream.Materialized - -/** 1. MAIN ABSTRACTION */ -trait AgnosticJoiner[AS[_, _], K1, V3] { - - def join[K2, V1, V2]( - keyExtractor: V1 => K2, - combiner: (V1, V2) => V3 - )( - as: AS[K1, V1], - bs: AS[K2, V2] - ): AS[K1, V3] - -} - -/** 2. DATA STRUCTURE SIMILAR TO KTABLE[K, V] */ -case class KTableT[K, V](items: Iterable[(K, V)]) - -object AgnosticJoiner { - - /** 3. PLACE WHERE WE DECOUPLE IMPLEMENTATION */ - def join[AS[_, _], K1, V1, K2, V2, V3]( - keyExtractor: V1 => K2, - combiner: (V1, V2) => V3 - )( - as: AS[K1, V1], - bs: AS[K2, V2] - )(implicit impl: AgnosticJoiner[AS, K1, V3]): AS[K1, V3] = - impl.join(keyExtractor, combiner)(as, bs) - - type KMaterialized[K1, V3] = Materialized[K1, V3, ByteArrayKeyValueStore] - - class KafkaStreamsJoiner[K1, V3](materialized: KMaterialized[K1, V3]) extends AgnosticJoiner[KTable, K1, V3] { - override def join[K2, V1, V2](keyExtractor: V1 => K2, combiner: (V1, V2) => V3)( - as: KTable[K1, V1], - bs: KTable[K2, V2] - ): KTable[K1, V3] = as.leftJoin(bs, keyExtractor, combiner(_, _), materialized) - } - - class IterableJoiner[K1, V3] extends AgnosticJoiner[KTableT, K1, V3] { - override def join[K2, V1, V2](keyExtractor: V1 => K2, combiner: (V1, V2) => V3)( - as: KTableT[K1, V1], - bs: KTableT[K2, V2] - ): KTableT[K1, V3] = KTableT( - as.items.flatMap { case (k1, v1) => - bs.items - .filter { case (k2, _) => k2 == keyExtractor(v1) } - .map { case (_, v2) => k1 -> combiner(v1, v2) } - } - ) - } - - class HashMapJoiner[K1, V3] extends AgnosticJoiner[Map, K1, V3] { - override def join[K2, V1, V2](keyExtractor: V1 => K2, combiner: (V1, V2) => V3)(as: Map[K1, V1], bs: Map[K2, V2]): Map[K1, V3] = ??? - } - -} - -/** 4.1. INSTANCE TO DECLARE IN CODE */ -object KafkaKTableInstances { - implicit def ksJoiner[K1, V3](implicit km: KMaterialized[K1, V3]): AgnosticJoiner[KTable, K1, V3] = new KafkaStreamsJoiner(km) -} - -/** 4.2. INSTANCE TO DECLARE IN TESTS */ -object TestKTableTInstances { - implicit def itJoiner[K1, V3]: AgnosticJoiner[KTableT, K1, V3] = new IterableJoiner[K1, V3] -} - -/** 4.3. INSTANCE TO DECLARE IN TESTS */ -object TestMapInstances { - implicit def mJoiner[K1, V3]: AgnosticJoiner[Map, K1, V3] = new HashMapJoiner[K1, V3] -} - -/** 5. BUSINESS LOGIC DECOUPLED */ -object BusinessLogicDefinedAgnostically { - - /** V1 => K2 */ - val keyExtractor: ((String, Long)) => Long = - (v1: (String, Long)) => v1._2 - - /** (V1, V2) => V3 */ - val valueCombiner: ((String, Long), (String, Double)) => (String, String, Double) = - (v1: (String, Long), v2: (String, Double)) => (v1._1, v2._1, v2._2) - -} - -/** 6.1 USE IN TESTS */ -object UsingInTests extends App { - import BusinessLogicDefinedAgnostically._ - import TestKTableTInstances._ - - val t1: KTableT[Int, (String, Long)] = KTableT( - Vector( - (1, ("test 1st", 101L)), - (2, ("test 2nd", 102L)), - (3, ("test 3rd", 103L)) - ) - ) - - val t2: KTableT[Long, (String, Double)] = KTableT( - Vector( - (101L, ("RED", 1001.1)), - (102L, ("GREEN", 1002.2)), - (103L, ("BLUE", 1003.3)) - ) - ) - - val r: KTableT[Int, (String, String, Double)] = - AgnosticJoiner.join(keyExtractor, valueCombiner)(t1, t2) - - r.items.foreach(println) -} - -/** 6.2 USE WITH KAFKA STREAMS */ -object UsingWithKafkaStreams { - import BusinessLogicDefinedAgnostically._ - import KafkaKTableInstances._ - - implicit val mat: KMaterialized[Int, (String, String, Double)] = ??? - val t1: KTable[Int, (String, Long)] = ??? - val t2: KTable[Long, (String, Double)] = ??? - - val r: KTable[Int, (String, String, Double)] = - AgnosticJoiner.join(keyExtractor, valueCombiner)(t1, t2) -} diff --git a/avro101/src/main/scala/ks/KafkaStreamLearning.scala b/avro101/src/main/scala/ks/KafkaStreamLearning.scala deleted file mode 100644 index 1556ffed..00000000 --- a/avro101/src/main/scala/ks/KafkaStreamLearning.scala +++ /dev/null @@ -1,39 +0,0 @@ -package ks - -import org.apache.kafka.streams.kstream.ValueJoiner -import org.apache.kafka.streams.scala.ByteArrayKeyValueStore -import org.apache.kafka.streams.scala.kstream.{KTable, Materialized} - -class JoinSignatureIfKeysSame[K, A, B](t1: KTable[K, A], t2: KTable[K, B]) { - - val x: KTable[K, (A, B)] = t1.leftJoin(t2)((a, b) => (a, b)) - -} - -/** this: KTable[K, V] - * - * def join[VO, VR] (other: KTable[K, VO]) (joiner: (V, VO) => VR): KTable[K, VR] - * def join[VO, VR] (other: KTable[K, VO], materialized: Materialized[K, VR, ByteArrayKeyValueStore])(joiner: (V, VO) => VR): KTable[K, VR] - * def join[VO, VR] (other: KTable[K, VO], named: Named) (joiner: (V, VO) => VR): KTable[K, VR] - * def join[VO, VR] (other: KTable[K, VO], named: Named, materialized: Materialized[K, VR, ByteArrayKeyValueStore])(joiner: (V, VO) => VR): KTable[K, VR] - * - * def leftJoin[VO, VR] (other: KTable[K, VO]) (joiner: (V, VO) => VR): KTable[K, VR] - * def leftJoin[VO, VR] (other: KTable[K, VO], named: Named) (joiner: (V, VO) => VR): KTable[K, VR] - * def leftJoin[VO, VR] (other: KTable[K, VO], named: Named, materialized: Materialized[K, VR, ByteArrayKeyValueStore])(joiner: (V, VO) => VR): KTable[K, VR] - * def leftJoin[VO, VR] (other: KTable[K, VO], materialized: Materialized[K, VR, ByteArrayKeyValueStore])(joiner: (V, VO) => VR): KTable[K, VR] - * - * + def leftJoin[VR, KO, VO](other: KTable[KO, VO], kx: Function[V, KO], joiner: ValueJoiner[V, VO, VR], materialized: Materialized[K, VR, ByteArrayKeyValueStore]): KTable[K, VR] - * def leftJoin[VR, KO, VO](other: KTable[KO, VO], kx: Function[V, KO], joiner: ValueJoiner[V, VO, VR], tableJoined: TableJoined[K, KO], materialized: Materialized[K, VR, ByteArrayKeyValueStore]): KTable[K, VR] - * def leftJoin[VR, KO, VO](other: KTable[KO, VO], kx: Function[V, KO], joiner: ValueJoiner[V, VO, VR], named: Named, materialized: Materialized[K, VR, ByteArrayKeyValueStore]): KTable[K, VR] - */ -class JoinSignatureIfKeysDifferent[K1, K2, A, B, C]( - t1: KTable[K1, A], - t2: KTable[K2, B], - keyExtractor: A => K2, - valueJoiner: ValueJoiner[A, B, C], - materialized: Materialized[K1, C, ByteArrayKeyValueStore] -) { - - val r: KTable[K1, C] = t1.leftJoin(t2, keyExtractor, valueJoiner, materialized) - -} diff --git a/avro101/src/main/scala/testjavainterop/BiFunc.java b/avro101/src/main/scala/testjavainterop/BiFunc.java deleted file mode 100644 index 230a38eb..00000000 --- a/avro101/src/main/scala/testjavainterop/BiFunc.java +++ /dev/null @@ -1,5 +0,0 @@ -package testjavainterop; - -public interface BiFunc { - C make(A a, B b); -} diff --git a/avro101/src/main/scala/testjavainterop/JavaScalaApp.scala b/avro101/src/main/scala/testjavainterop/JavaScalaApp.scala deleted file mode 100644 index 913ab24c..00000000 --- a/avro101/src/main/scala/testjavainterop/JavaScalaApp.scala +++ /dev/null @@ -1,26 +0,0 @@ -package testjavainterop - -object JavaScalaApp extends App { - - def doSomething(f: BiFunc[Int, Int, Int])(a: Int, b: Int) = - f.make(a, b) - - /** lambda will be lifted automatically to Java Interface */ - val x: Int = doSomething((a, b) => a + b)(1, 2) - println(x) - - /** method reference will be lifted automatically to Java Interface */ - def f1(a: Int, b: Int) = a + b - val y: Int = doSomething(f1)(1, 2) - println(y) - - val f2 = (a: Int, b: Int) => a + b - - /** Scala f: (Int, Int) => Int can't be lifted to Java interface */ - // val y: Int = doSomething(f2)(1, 2) - /** needs to be rewritten */ - val jf2: BiFunc[Int, Int, Int] = (a: Int, b: Int) => f2(a, b) - val z: Int = doSomething(jf2)(1, 2) - println(z) - -} diff --git a/avro101/src/main/scala/tools/Console.scala b/avro101/src/main/scala/tools/Console.scala deleted file mode 100644 index a8777846..00000000 --- a/avro101/src/main/scala/tools/Console.scala +++ /dev/null @@ -1,7 +0,0 @@ -package tools - -object Console { - - def delimiter() = println("-" * 50) - -} diff --git a/avro101/src/main/scala/tools/Tools.scala b/avro101/src/main/scala/tools/Tools.scala deleted file mode 100644 index 4e391bf1..00000000 --- a/avro101/src/main/scala/tools/Tools.scala +++ /dev/null @@ -1,59 +0,0 @@ -package tools - -import cats.implicits.catsSyntaxOptionId -import cats.implicits.toBifunctorOps -import cats.implicits.toFunctorOps -import java.io.ByteArrayOutputStream -import java.io.File -import org.apache.avro.Schema -import org.apache.avro.file.DataFileReader -import org.apache.avro.file.DataFileWriter -import org.apache.avro.generic.GenericDatumReader -import org.apache.avro.generic.GenericDatumWriter -import org.apache.avro.generic.GenericRecord -import scala.collection.JavaConverters._ -import scala.util.Try - -object Tools { - - def validateRecord(record: GenericRecord) = { - val schema = record.getSchema - val gw = new GenericDatumWriter[GenericRecord](schema) - val w = new DataFileWriter[GenericRecord](gw) - val os = new ByteArrayOutputStream() - w.create(schema, os) - val x = Try(w.append(record)).toEither.leftMap(_.getCause.getMessage).as(record) - w.close() - x - } - - /** write to file with schema */ - def writeTo(file: File, schema: Schema, items: GenericRecord*) = { - val gw = new GenericDatumWriter[GenericRecord](schema) - val w = new DataFileWriter[GenericRecord](gw) - w.create(schema, file) - items.foreach(w.append) - w.close() - } - - private def readFrom(file: File, maybeSchema: Option[Schema]): List[GenericRecord] = { - val gr = maybeSchema.fold( - new GenericDatumReader[GenericRecord] - ) { schema => - new GenericDatumReader[GenericRecord](schema) - } - val r = new DataFileReader[GenericRecord](file, gr) - val x = r.iterator().asScala.toList - r.close() - x - } - - /** read without schema check enforced */ - def readFrom(file: File): List[GenericRecord] = - readFrom(file, None) - - /** read WITH schema check enforced */ - def readFrom(file: File, schema: Schema): List[GenericRecord] = - readFrom(file, schema.some) - -} diff --git a/avro101/src/main/scala/tools/Utils.scala b/avro101/src/main/scala/tools/Utils.scala deleted file mode 100644 index 088676d4..00000000 --- a/avro101/src/main/scala/tools/Utils.scala +++ /dev/null @@ -1,18 +0,0 @@ -package tools - -import java.io.File -import java.nio.file.{Files, Paths} - -object Utils { - - def absolutePathFromResources(name: String): String = - getClass.getClassLoader.getResource(name).getFile - - def fileFromResources(name: String): File = - new File(absolutePathFromResources(name)) - - // JDK 11 - def contentsFromResources(name: String): String = ??? -// Files.readString(Paths.get(absolutePathFromResources(name))) - -} diff --git a/avro101/todo.md b/avro101/todo.md deleted file mode 100644 index 379160fb..00000000 --- a/avro101/todo.md +++ /dev/null @@ -1,11 +0,0 @@ -### AVRO is a file format like JSOn, but with Schema attached - -- create schema programmatically -- create schema by parsing string -- create schema by reading from file -- write file -- read file -- read file with schema -- analyze behaviour wuth wrong properties names -- analyze behaviour wuth wrong properties types -- analyze behaviour wuth nullable / missing properties diff --git a/build.sbt b/build.sbt index 7183d4b5..92a310e2 100644 --- a/build.sbt +++ b/build.sbt @@ -10,35 +10,22 @@ lazy val sandbox = (project in file("sandbox")) Settings.common2, description := "sandbox, cat effects 3 based", libraryDependencies ++= Seq( - "co.fs2" %% "fs2-core" % "3.10.2", - "co.fs2" %% "fs2-io" % "3.10.2", + "co.fs2" %% "fs2-core" % "3.10.2", + "co.fs2" %% "fs2-io" % "3.10.2", // enum - "com.beachape" %% "enumeratum" % "1.7.4", - "com.beachape" %% "enumeratum-circe" % "1.7.4", - "com.beachape" %% "enumeratum-cats" % "1.7.4", - "com.beachape" %% "enumeratum-scalacheck" % "1.7.4", + "com.beachape" %% "enumeratum" % "1.7.4", + "com.beachape" %% "enumeratum-circe" % "1.7.4", + "com.beachape" %% "enumeratum-cats" % "1.7.4", + "com.beachape" %% "enumeratum-scalacheck" % "1.7.4", // json - "io.circe" %% "circe-parser" % "0.14.9", - "io.circe" %% "circe-generic-extras" % "0.14.3", - "io.circe" %% "circe-fs2" % "0.14.1", + "io.circe" %% "circe-parser" % "0.14.9", + "io.circe" %% "circe-generic-extras" % "0.14.4", + "io.circe" %% "circe-fs2" % "0.14.1", + // twiddles + "org.typelevel" %% "twiddles-core" % "0.9.0", ), ) -lazy val avro101 = (project in file("avro101")) - .settings( - scalaVersion := v.vScala212, - javacOptions ++= CompilerOptions.javacOptions, - scalacOptions ++= CompilerOptions.scalacOptions, - scalacOptions -= ScalacOpts.warningsAsFatals, // we are learning, there is no sense to be so strict - scalacOptions -= ScalacOpts.macroAnnotations, // 2.12 doesn't have it - libraryDependencies ++= Libraries.testingToolkit2, - libraryDependencies ++= Seq( - Libraries.cats, - "org.apache.avro" % "avro" % "1.11.3", - "org.apache.kafka" %% "kafka-streams-scala" % "3.7.0", - ) - ) - lazy val mono101 = (project in file("mono101")) .settings( Settings.common2, @@ -119,7 +106,7 @@ lazy val ce3 = (project in file("ce3")) libraryDependencies ++= Seq( // core "org.typelevel" %% "cats-core" % "2.12.0", - "org.typelevel" %% "cats-effect" % "3.5.3", + "org.typelevel" %% "cats-effect" % "3.5.4", // stm "io.github.timwspence" %% "cats-stm" % "0.13.5", // streams @@ -129,13 +116,13 @@ lazy val ce3 = (project in file("ce3")) // enum "com.beachape" %% "enumeratum" % "1.7.3", "com.beachape" %% "enumeratum-circe" % "1.7.3", - "com.beachape" %% "enumeratum-doobie" % "1.7.3", // 1.7.4 brings doobie 1.0.0-RC4 "com.beachape" %% "enumeratum-cats" % "1.7.3", + "com.beachape" %% "enumeratum-doobie" % "1.7.3", // 1.7.4 brings doobie 1.0.0-RC4 "com.beachape" %% "enumeratum-scalacheck" % "1.7.3", // json - "io.circe" %% "circe-parser" % "0.14.7", + "io.circe" %% "circe-parser" % "0.14.9", "io.circe" %% "circe-optics" % "0.14.1", // 0.15.0 - "io.circe" %% "circe-generic-extras" % "0.14.3", + "io.circe" %% "circe-generic-extras" % "0.14.4", "io.circe" %% "circe-yaml" % "0.14.2", // 0.15.1 "io.circe" %% "circe-fs2" % "0.14.1", "io.circe" %% "circe-shapes" % "0.14.7", @@ -157,10 +144,10 @@ lazy val ce3 = (project in file("ce3")) "org.http4s" %% "http4s-blaze-server" % "0.23.16", "org.http4s" %% "http4s-blaze-client" % "0.23.16", // tapir - "com.softwaremill.sttp.tapir" %% "tapir-core" % "1.10.9", - "com.softwaremill.sttp.tapir" %% "tapir-json-circe" % "1.10.8", - "com.softwaremill.sttp.tapir" %% "tapir-http4s-server" % "1.10.8", - "org.scalameta" %% "scalameta" % "4.9.4", + "com.softwaremill.sttp.tapir" %% "tapir-core" % "1.11.1", + "com.softwaremill.sttp.tapir" %% "tapir-json-circe" % "1.11.1", + "com.softwaremill.sttp.tapir" %% "tapir-http4s-server" % "1.11.1", + "org.scalameta" %% "scalameta" % "4.9.9", // other "io.kubernetes" % "client-java-api" % "20.0.1", "io.kubernetes" % "client-java" % "20.0.1", @@ -168,8 +155,9 @@ lazy val ce3 = (project in file("ce3")) "io.scalaland" %% "chimney" % "1.1.0", "org.tpolecat" %% "skunk-core" % "0.6.4", "io.7mind.izumi" %% "logstage-core" % "1.2.10", - Libraries.doobieCore, - Libraries.doobiePg, + "org.tpolecat" %% "doobie-core" % "1.0.0-RC2", + "org.tpolecat" %% "doobie-postgres" % "1.0.0-RC2", + "org.tpolecat" %% "doobie-hikari" % "1.0.0-RC2", Libraries.sqlPostgres, Libraries.newtype, Libraries.refinedCore, @@ -205,17 +193,17 @@ lazy val httpt = (project in file("httpt")) Settings.common2, description := "HTTP load tests", libraryDependencies ++= Seq( - "org.http4s" %% "http4s-dsl" % "1.0.0-M39", - "org.http4s" %% "http4s-circe" % "1.0.0-M39", - "org.http4s" %% "http4s-ember-server" % "1.0.0-M39", // 13.07.2023 - "org.http4s" %% "http4s-blaze-server" % "1.0.0-M38", // 4.01.2023 + "org.http4s" %% "http4s-dsl" % "1.0.0-M41", + "org.http4s" %% "http4s-circe" % "1.0.0-M41", + "org.http4s" %% "http4s-ember-server" % "1.0.0-M41", + "org.http4s" %% "http4s-blaze-server" % "1.0.0-M40", "org.http4s" %% "http4s-jetty-server" % "1.0.0-M32", "org.typelevel" %% "log4cats-core" % "2.7.0", "org.typelevel" %% "log4cats-slf4j" % "2.7.0" ), dependencyOverrides ++= Seq( - "org.http4s" %% "http4s-core" % "1.0.0-M39", - "org.http4s" %% "http4s-server" % "1.0.0-M39" + "org.http4s" %% "http4s-core" % "1.0.0-M41", + "org.http4s" %% "http4s-server" % "1.0.0-M41" ) ) @@ -327,15 +315,6 @@ lazy val lihaoyi = (project in file("lihaoyi")) ) ) -lazy val mix = (project in file("mix")) - .settings( - Settings.common2, - libraryDependencies ++= Seq( - "org.flywaydb" % "flyway-core" % "6.4.2", - Libraries.jsoup, - ), - ) - /** protobuf experiments */ lazy val pbx = (project in file("pbx")) .enablePlugins(ScalaxbPlugin) @@ -350,7 +329,7 @@ lazy val plain2 = (project in file("plain2")) .settings( Settings.common2, libraryDependencies ++= Seq( - "org.typelevel" %% "cats-core" % "2.10.0", + "org.typelevel" %% "cats-core" % "2.12.0", "io.chymyst" %% "curryhoward" % "0.3.8", "com.softwaremill.quicklens" %% "quicklens" % "1.9.7", ) @@ -385,7 +364,7 @@ lazy val typesafe = (project in file("typesafe")) "com.typesafe.slick" %% "slick-hikaricp" % "3.4.1", "com.typesafe.play" %% "play-json" % "2.9.4", "com.typesafe" % "config" % "1.4.2", - "ch.qos.logback" % "logback-classic" % "1.4.7" + "ch.qos.logback" % "logback-classic" % "1.5.7" ) ) @@ -413,8 +392,8 @@ lazy val spring = (project in file("spring")) description := "SpringBoot Experiments", Settings.common2, libraryDependencies ++= Seq( - "org.projectlombok" % "lombok" % "1.18.32", - "org.springframework.boot" % "spring-boot-starter-web" % "3.2.6", + "org.projectlombok" % "lombok" % "1.18.34", + "org.springframework.boot" % "spring-boot-starter-web" % "3.3.2", ), ) @@ -426,9 +405,9 @@ lazy val http4sws = (project in file("http4sws")) "org.http4s" %% "http4s-dsl" % "0.23.27", "org.http4s" %% "http4s-circe" % "0.23.27", "org.http4s" %% "http4s-ember-server" % "0.23.27", - "io.circe" %% "circe-generic" % "0.14.7", + "io.circe" %% "circe-generic" % "0.14.9", "org.typelevel" %% "cats-parse" % "1.0.0", - "ch.qos.logback" % "logback-classic" % "1.5.6", + "ch.qos.logback" % "logback-classic" % "1.5.7", ) ) diff --git a/ce3/src/main/scala/_sandbox/SandboxTest.scala b/ce3/src/main/scala/_sandbox/SandboxTest.scala index fc3162a5..eeb96137 100644 --- a/ce3/src/main/scala/_sandbox/SandboxTest.scala +++ b/ce3/src/main/scala/_sandbox/SandboxTest.scala @@ -8,24 +8,6 @@ import org.scalatestplus.scalacheck.ScalaCheckPropertyChecks class SandboxTest extends AnyFunSuite with Matchers with ScalaCheckPropertyChecks with Inside { - def method1: Int = ??? - def method2: String = ??? - - class MyTuple(val x: Int, val name: String) - - def method3: (Int, String) = { - return (33, "hello") - } - - def method4: MyTuple = { - return new MyTuple(33, "hello") - } - - val r1: (Int, String) = method3 - val (x: Int, y: String) = method3 - - - test("1") { } diff --git a/ce3/src/main/scala/fss101/Fs2ReactivePlayground.scala b/ce3/src/main/scala/fss101/Fs2ReactivePlayground.scala deleted file mode 100644 index 00836f16..00000000 --- a/ce3/src/main/scala/fss101/Fs2ReactivePlayground.scala +++ /dev/null @@ -1,32 +0,0 @@ -package fs2x - -import cats.effect.Async -import fs2.interop.reactivestreams._ -import org.reactivestreams.Publisher -import org.reactivestreams.Subscriber - -import java.util.concurrent.Flow - -/** ReactivePlayground - * - * - consuming from org.reactivestreams.Publisher[A] - * to fs2.Stream[F, A] - * - * - publishing to org.reactivestreams.Publisher[A] - * via fs2Stream.subscribe(elasticSearchSubscriber) - */ -class Fs2ReactivePlayground[F[_]: Async, A] { - - /** having any library providing source as a [[org.reactivestreams.Publisher]] */ - val elasticSearchPublisher: Publisher[A] = ??? - - /** we can easily convert it to fs2Stream */ - val fs2Stream: fs2.Stream[F, A] = fromPublisher[F, A](elasticSearchPublisher, 1000) - - /** having any library providing source as a [[org.reactivestreams.Subscriber]] */ - val elasticSearchSubscriber: Flow.Subscriber[A] = ??? - - /** we can sink top it */ - val r = fs2Stream.subscribe(elasticSearchSubscriber) - -} diff --git a/ce3/src/main/scala/fss101/d12sigterm/SigTermRefApp.scala b/ce3/src/main/scala/fss101/d12sigterm/SigTermRefApp.scala new file mode 100644 index 00000000..453057d4 --- /dev/null +++ b/ce3/src/main/scala/fss101/d12sigterm/SigTermRefApp.scala @@ -0,0 +1,64 @@ +package fss101.d12sigterm + +import cats.effect.Deferred +import cats.effect.IO +import cats.effect.IOApp +import cats.effect.Ref +import cats.implicits.catsSyntaxEitherId +import scala.concurrent.duration.DurationInt +import scala.concurrent.duration.FiniteDuration + +object SigTermRefApp extends IOApp.Simple { + + val hook: IO[Deferred[IO, Unit]] = IO.deferred[Unit] + + def shutdownHook = hook + .flatMap(_.complete(())) + .unsafeRunSync()(cats.effect.unsafe.implicits.global) + + sys.addShutdownHook(shutdownHook) + + val haltSignal: IO[Either[Throwable, Unit]] = + hook + .flatMap(_.get) + .map(_.asRight[Throwable]) + + /** allocate cell */ + def mkRef: IO[Ref[IO, List[FiniteDuration]]] = + IO.ref(List.empty[FiniteDuration]) + + /** collect data to cell */ + def collect(d: FiniteDuration)(ref: Ref[IO, List[FiniteDuration]]): IO[Unit] = + ref.update(d :: _) + + /** report all things collected */ + def report(ref: Ref[IO, List[FiniteDuration]]): IO[Unit] = + ref.get.flatMap { ds => + IO.blocking { + println("collected:") + ds.foreach(x => pprint.log(x)) + } + } + + def infiniteStream(ref: Ref[IO, List[FiniteDuration]]): fs2.Stream[IO, Nothing] = + fs2.Stream + .awakeEvery[IO](1.second) + .evalTap(x => IO.println(x)) + .evalTap(x => collect(x)(ref)) + .drain + + override def run: IO[Unit] = + mkRef + .flatMap { ref => + + val finalizer = + IO.println("finally") >> report(ref) + + infiniteStream(ref) + .interruptWhen(haltSignal) + .onFinalize(finalizer) + .compile + .drain + } + +} diff --git a/ce3/src/main/scala/fss101/d12sigterm/StreamPreciseTerminationControl.scala b/ce3/src/main/scala/fss101/d12sigterm/StreamPreciseTerminationControl.scala new file mode 100644 index 00000000..c0e9c0b7 --- /dev/null +++ b/ce3/src/main/scala/fss101/d12sigterm/StreamPreciseTerminationControl.scala @@ -0,0 +1,29 @@ +package fss101.d12sigterm + +import cats.effect.IO +import cats.effect.IOApp +import cats.effect.kernel.Resource.ExitCase +import cats.implicits._ +import scala.concurrent.duration.DurationInt + +object StreamPreciseTerminationControl extends IOApp.Simple { + + def infiniteStream = fs2.Stream + .awakeEvery[IO](1.second) + // .take(4) + .evalTap(IO.println) + .drain + + override def run: IO[Unit] = + infiniteStream + // .timeout(5.second) + .onFinalizeCase { + // precise termination control + case ExitCase.Succeeded => IO.println("Succeeded - Just finished") // Just finished if finish before timeout + case ExitCase.Errored(e) => IO.println(s"Errored - With exception (any unhandled): $e") // if finished due to timeout + case ExitCase.Canceled => IO.println("Canceled - somehow (maybe SIGTERM/SIGINT)") // if cancelled + } + .compile + .drain + +} diff --git a/ce3/src/main/scala/fss101/d12sigterm/StreamTerminationFinalizer.scala b/ce3/src/main/scala/fss101/d12sigterm/StreamTerminationFinalizer.scala new file mode 100644 index 00000000..3d8860c4 --- /dev/null +++ b/ce3/src/main/scala/fss101/d12sigterm/StreamTerminationFinalizer.scala @@ -0,0 +1,23 @@ +package fss101.d12sigterm + +import cats.effect.IO +import cats.effect.IOApp +import cats.implicits._ +import scala.concurrent.duration.DurationInt + +object StreamTerminationFinalizer extends IOApp.Simple { + + def infiniteStream = fs2.Stream + .awakeEvery[IO](1.second) +// .take(4) + .evalTap(IO.println) + .drain + + override def run: IO[Unit] = + infiniteStream +// .timeout(5.second) + .onFinalize(IO.println("finally...")) // in the end of the stream regardless the case + .compile + .drain + +} diff --git a/ce3/src/main/scala/fss101/d12sigterm/StreamTerminationPlayground.scala b/ce3/src/main/scala/fss101/d12sigterm/StreamTerminationPlayground.scala new file mode 100644 index 00000000..5bd8e8bb --- /dev/null +++ b/ce3/src/main/scala/fss101/d12sigterm/StreamTerminationPlayground.scala @@ -0,0 +1,65 @@ +package fss101.d12sigterm + +import cats.effect.Deferred +import cats.effect.IO +import cats.effect.IOApp +import cats.effect.kernel.Resource.ExitCase +import cats.implicits._ +import scala.concurrent.duration.DurationInt + +object StreamTerminationPlayground extends IOApp.Simple { + + val latch: IO[Deferred[IO, Either[Throwable, Unit]]] = IO.deferred[Either[Throwable, Unit]] + + def hook = latch + .flatTap(_ => IO.println("2. Terminating JVM (maybe due to SIGINT/SIGTERM)")) + .flatMap(d => d.complete(().asRight)) // success termination ??? +// .flatMap(_.complete(new IllegalArgumentException("boom!!!").asLeft)) // failed termination ??? + .flatTap(_ => IO.println("3. Deferred completed")) + .unsafeRunSync()(cats.effect.unsafe.implicits.global) + + /** JVM platform-related */ + sys.addShutdownHook(hook) + + val haltSignal: IO[Either[Throwable, Unit]] = + latch + .flatTap(_ => IO.println("1. Waiting for Deferred to be completed")) + .flatMap(_.get) + + def infiniteStream = fs2.Stream + .awakeEvery[IO](1.second) +// .take(4) + .evalTap(IO.println) + .drain + + /* + we can interrupt ??? + - with error - Left(Throwable) + - without error - Right(Unit) + + // 1a. based on the F[Either[E, Unit]] + def interruptWhen(haltOnSignal: F[Either[Throwable, Unit]] ): Stream[F, O] + + // 1b. based on the Deferred resolved to F[Either[E, Unit]] + def interruptWhen(haltWhenTrue: Deferred[F, Either[Throwable, Unit]]): Stream[F, O] + + // 2a. based on Stream contains "true" + def interruptWhen(haltWhenTrue: Stream[F, Boolean] ): Stream[F, O] + + // 2b. based on Signal contains "true" + def interruptWhen(haltWhenTrue: Signal[F2, Boolean] ): Stream[F, O] + */ + override def run: IO[Unit] = + infiniteStream + .timeout(5.second) // ExitCase.Errored + .interruptWhen(haltSignal) + .onFinalizeCase { + case ExitCase.Succeeded => IO.println("Succeeded - Just finished") // Just finished + case ExitCase.Errored(e) => IO.println(s"Errored - With exception (any unhandled): $e") + case ExitCase.Canceled => IO.println("4. Canceled - via .interruptWhen()") + } + .onFinalize(IO.println("9. finally")) // in the end of the stream regardless the case + .compile + .drain + +} diff --git a/mix/src/main/scala/JsoupApi.scala b/ce3/src/main/scala/jsoupapi/JsoupApi.scala similarity index 94% rename from mix/src/main/scala/JsoupApi.scala rename to ce3/src/main/scala/jsoupapi/JsoupApi.scala index afa06e41..179bc133 100644 --- a/mix/src/main/scala/JsoupApi.scala +++ b/ce3/src/main/scala/jsoupapi/JsoupApi.scala @@ -1,4 +1,4 @@ -package hr_parse +package jsoupapi import org.jsoup.select.Elements diff --git a/ce3/src/main/scala/lit/Port.scala b/ce3/src/main/scala/lit/Port.scala new file mode 100644 index 00000000..98aa0ba4 --- /dev/null +++ b/ce3/src/main/scala/lit/Port.scala @@ -0,0 +1,31 @@ +package lit + +import org.typelevel.literally.Literally + +case class Port(value: Int) extends AnyVal + +object Port { + /** code to validate */ + def fromString(raw: String): Either[String, Port] = + Option(raw) + .flatMap(_.toIntOption) + .filter(_ > 0) + .filter(_ <= 65535) + .map(Port.apply) + .toRight("invalid port - must be integer between 0 and 65535") + + object PortLiteral extends Literally[Port] { + def validate(c: Context)(s: String): Either[String, c.Expr[Port]] = { + import c.universe._ + Port.fromString(s) + .map(_ => c.Expr(q"Port.fromString($s).get")) + } + + def make(c: Context)(args: c.Expr[Any]*): c.Expr[Port] = this.apply(c)(args: _*) + } + + implicit class port(val sc: StringContext) extends AnyVal { + def port(args: Any*): Port = macro PortLiteral.make + } + +} diff --git a/complement/repositories.md b/complement/repositories.md index 3579c890..95eb23b4 100644 --- a/complement/repositories.md +++ b/complement/repositories.md @@ -30,6 +30,7 @@ - git@github.com:djnzx/concurrency-in-practice-java.git - git@github.com:djnzx/learning-scala3.git - git@github.com:djnzx/scala-99.git +- git@github.com:djnzx/scala-avro-kafka-flows.git ### others - notable! - Adam Warski (https://github.com/adamw) diff --git a/httpt/src/main/scala/app/BlazeServer.scala b/httpt/src/main/scala/app/BlazeServer.scala index 93ba711b..93f9dd8c 100644 --- a/httpt/src/main/scala/app/BlazeServer.scala +++ b/httpt/src/main/scala/app/BlazeServer.scala @@ -2,9 +2,12 @@ package app import cats.effect.IO import org.http4s.blaze.server.BlazeServerBuilder +import org.typelevel.log4cats.slf4j.Slf4jFactory object BlazeServer extends MyIOApp { + implicit val logger = Slf4jFactory.create[IO] + override def run: IO[Unit] = BlazeServerBuilder[IO] .bindHttp(8080, "0.0.0.0") .withHttpApp(TestRoutes[IO].orNotFound) diff --git a/mix/src/main/resources/db/migration/V2020.05.14.01__drop_if_exists.sql b/mix/src/main/resources/db/migration/V2020.05.14.01__drop_if_exists.sql deleted file mode 100644 index b04e09dc..00000000 --- a/mix/src/main/resources/db/migration/V2020.05.14.01__drop_if_exists.sql +++ /dev/null @@ -1,4 +0,0 @@ -drop table if exists ranks_current; -drop table if exists ranks; -drop table if exists students; -drop table if exists groups; diff --git a/mix/src/main/resources/db/migration/V2020.05.14.02__schema_tab.sql b/mix/src/main/resources/db/migration/V2020.05.14.02__schema_tab.sql deleted file mode 100644 index 2e45bc2d..00000000 --- a/mix/src/main/resources/db/migration/V2020.05.14.02__schema_tab.sql +++ /dev/null @@ -1,40 +0,0 @@ -create table groups ( - id serial - constraint groups_pk - primary key, - name varchar not null -); - -create table students ( - id serial - constraint students_pk - primary key, - name varchar not null, - hacker varchar not null, - groupp integer - constraint students_fk1 - references groups -); - -create table ranks ( - id bigserial - constraint ranks_pk - primary key, - date timestamp default now(), - student integer not null - constraint ranks_fk1 - references students, - rank integer not null, - score double precision not null -); - -create table ranks_current ( - student integer not null - constraint ranks_current_pk - primary key - constraint ranks_current_fk1 - references students, - rank integer, - score double precision, - date timestamp -); diff --git a/mix/src/main/resources/db/migration/V2020.05.14.03__schema_view.sql b/mix/src/main/resources/db/migration/V2020.05.14.03__schema_view.sql deleted file mode 100644 index eab02a21..00000000 --- a/mix/src/main/resources/db/migration/V2020.05.14.03__schema_view.sql +++ /dev/null @@ -1,13 +0,0 @@ -drop view if exists v_ranks_current; -create view v_ranks_current(sid, sname, gid, gname, hacker, rank, score, date) as -SELECT s.id AS sid, - s.name AS sname, - g.id AS gid, - g.name AS gname, - s.hacker, - rc.rank, - rc.score, - rc.date -FROM students s - LEFT JOIN groups g ON s.groupp = g.id - LEFT JOIN ranks_current rc ON s.id = rc.student; diff --git a/mix/src/main/resources/db/migration/V2020.05.14.04__schema_trigger.sql b/mix/src/main/resources/db/migration/V2020.05.14.04__schema_trigger.sql deleted file mode 100644 index 48d4e189..00000000 --- a/mix/src/main/resources/db/migration/V2020.05.14.04__schema_trigger.sql +++ /dev/null @@ -1,17 +0,0 @@ -CREATE FUNCTION upd_current_rank() RETURNS trigger AS $$ -BEGIN - if (TG_OP = 'INSERT') then - insert into ranks_current (student, rank, score, date) - values (new.student, new.rank, new.score, new.date) - on conflict (student) do - update set rank=new.rank, score=new.score, date=new.date; - end if; - return new; -END; -$$ LANGUAGE plpgsql; - -create trigger ranks_ai - after insert - on ranks - for each row -execute procedure upd_current_rank(); diff --git a/mix/src/main/resources/db/migration/V2020.05.14.05__data.sql b/mix/src/main/resources/db/migration/V2020.05.14.05__data.sql deleted file mode 100644 index d4259bc3..00000000 --- a/mix/src/main/resources/db/migration/V2020.05.14.05__data.sql +++ /dev/null @@ -1,13 +0,0 @@ -INSERT INTO groups (name) VALUES ('BE1'); -- 1 -INSERT INTO groups (name) VALUES ('BE2'); -- 2 -INSERT INTO groups (name) VALUES ('BE3'); -- 3 -INSERT INTO groups (name) VALUES ('BE4'); -- 4 - -INSERT INTO students (name, hacker, groupp) VALUES ('Adil/Aqil Abdurrazaklı', 'abdurrazakliadil', 4); -- 1 -INSERT INTO students (name, hacker, groupp) VALUES ('Aflatun Velibeyli', 'a_velibeyli6', 4); -- 2 -INSERT INTO students (name, hacker, groupp) VALUES ('Intigam Nagiyev', 'nagiyevintiqam', 4); -- 3 -INSERT INTO students (name, hacker, groupp) VALUES ('Fidan Ismayilova', 'fidan97ismailova', 4); -- 4 -INSERT INTO students (name, hacker, groupp) VALUES ('Jalal Ibrahimkhalilov', 'ibcelal', 4); -- 5 - -INSERT INTO ranks (date, student, rank, score) VALUES ('2020-05-14 13:18:59.995967', 1, 1000, 50.6); -INSERT INTO ranks (date, student, rank, score) VALUES ('2020-05-14 13:19:55.605450', 1, 990, 51.5); diff --git a/mix/src/main/resources/db/migration/V2020.0521.01__table_modify.sql b/mix/src/main/resources/db/migration/V2020.0521.01__table_modify.sql deleted file mode 100644 index 43199e1f..00000000 --- a/mix/src/main/resources/db/migration/V2020.0521.01__table_modify.sql +++ /dev/null @@ -1 +0,0 @@ -alter table students alter column hacker drop not null; diff --git a/mix/src/main/resources/db/migration/V2020.0521.02__delete_data.sql b/mix/src/main/resources/db/migration/V2020.0521.02__delete_data.sql deleted file mode 100644 index aa047b3e..00000000 --- a/mix/src/main/resources/db/migration/V2020.0521.02__delete_data.sql +++ /dev/null @@ -1,3 +0,0 @@ -delete from ranks_current; -delete from ranks; -delete from students; diff --git a/mix/src/main/scala/FlyWayApi.scala b/mix/src/main/scala/FlyWayApi.scala deleted file mode 100644 index a8971550..00000000 --- a/mix/src/main/scala/FlyWayApi.scala +++ /dev/null @@ -1,18 +0,0 @@ -import org.flywaydb.core.Flyway -import org.flywaydb.core.api.configuration.FluentConfiguration - -trait FlyWayApi { - - val url = "jdbc:postgresql://localhost:5432/ibatech" - val username = "postgres" - val password = "secret" - - def dbSetup(clean: Boolean = false): Unit = { - val config = new FluentConfiguration() - .dataSource(url, username, password) - val flyway = new Flyway(config) - if (clean) flyway.clean() - flyway.migrate() - } - -} diff --git a/mix/src/main/scala/links.md b/mix/src/main/scala/links.md deleted file mode 100644 index 171a166a..00000000 --- a/mix/src/main/scala/links.md +++ /dev/null @@ -1,6 +0,0 @@ -- [quill](https://www.lihaoyi.com/post/WorkingwithDatabasesusingScalaandQuill.html) -- [date vs date](https://stackoverflow.com/questions/2305973/java-util-date-vs-java-sql-date) -- [encoders](https://github.com/getquill/quill/blob/master/quill-jdbc/src/main/scala/io/getquill/context/jdbc/Encoders.scala) -- -- [Jsoup main page](https://jsoup.org) -- [Jsoup select syntax](https://jsoup.org/cookbook/extracting-data/selector-syntax) diff --git a/munitx/src/main/scala/eff/EffectsTesting.scala b/munitx/src/main/scala/eff/EffectsTesting.scala new file mode 100644 index 00000000..6d6a9d7c --- /dev/null +++ b/munitx/src/main/scala/eff/EffectsTesting.scala @@ -0,0 +1,15 @@ +package eff + +import cats.effect._ +import munit.CatsEffectSuite +import skunkx.Tools + +class EffectsTesting extends CatsEffectSuite with Tools { + + test("1") { + IO.realTime + .map(_.toMillis) + .log + } + +} diff --git a/project/Libraries.scala b/project/Libraries.scala index d86cdfca..1bb49671 100644 --- a/project/Libraries.scala +++ b/project/Libraries.scala @@ -12,12 +12,6 @@ object Libraries { val catsLaws = "org.typelevel" %% "cats-laws" % "2.10.0" val catsMtl = "org.typelevel" %% "cats-mtl" % "1.3.1" val catsRetry = "com.github.cb372" %% "cats-retry" % "3.1.0" - // postgreSQL related - val doobieCore = "org.tpolecat" %% "doobie-core" % "1.0.0-RC2" - val doobiePg = "org.tpolecat" %% "doobie-postgres" % "1.0.0-RC2" - val doobieHikari = "org.tpolecat" %% "doobie-hikari" % "1.0.0-RC2" - val skunkCore = "org.tpolecat" %% "skunk-core" % "1.0.0-M1" // no more 2.12 - val skunkCirce = "org.tpolecat" %% "skunk-circe" % "1.0.0-M1" val sqlPostgres = "org.postgresql" % "postgresql" % "42.6.0" // val http4sJwtAuth = "dev.profunktor" %% "http4s-jwt-auth" % "1.2.1" diff --git a/project/Resolvers.scala b/project/Resolvers.scala index 123c854a..1d187e5d 100644 --- a/project/Resolvers.scala +++ b/project/Resolvers.scala @@ -1,11 +1,12 @@ -import sbt.Resolver +import sbt.* object Resolvers { val all = Seq( Resolver.mavenLocal, Resolver.mavenCentral, -// Resolver.typesafeRepo("releases"), + "confluent" at "https://packages.confluent.io/maven/", + Resolver.typesafeRepo("releases"), // Resolver.sonatypeRepo("releases"), // Resolver.sonatypeRepo("snapshots"), Repos.artima, diff --git a/project/build.properties b/project/build.properties index 081fdbbc..ee4c672c 100644 --- a/project/build.properties +++ b/project/build.properties @@ -1 +1 @@ -sbt.version=1.10.0 +sbt.version=1.10.1 diff --git a/sandbox/src/main/scala/twdl/TwiddlePlayground.scala b/sandbox/src/main/scala/twdl/TwiddlePlayground.scala new file mode 100644 index 00000000..dd69230e --- /dev/null +++ b/sandbox/src/main/scala/twdl/TwiddlePlayground.scala @@ -0,0 +1,44 @@ +package twdl + +import common.Base +import org.scalatest.Succeeded +import org.typelevel.twiddles._ +import org.typelevel.twiddles.syntax._ +import shapeless._ + +// https://github.com/typelevel/twiddles +object TwiddlePlayground { + + /** On Scala 3, twiddle lists are represented as generic tuples -- e.g., + * F[Int *: String *: Boolean *: EmptyTuple] or equivalently + * F[(Int, String, Boolean)]. + * + * On Scala 2, twiddle lists are represented as Shapeless heterogeneous lists. + */ + + val x = 1 + +} + +class TwiddlePlayground extends Base { + + import TwiddlePlayground._ + + test("1") { + + case class Foo(x: Int, y: String) + + val a = Option(42) + val b = Option("Hi") + + val ab: Option[Int *: String *: EmptyTuple] = a *: b + val foo: Option[Foo] = ab.to[Foo] + + val abh: Int :: String :: EmptyTuple = 42 :: "Hi" :: HNil + + pprint.log(ab) + pprint.log(abh) + pprint.log(foo) + } + +}