diff --git a/.github/workflows/benchmark_group_2.yml b/.github/workflows/benchmark_group_2.yml index 3346199..019ba7e 100644 --- a/.github/workflows/benchmark_group_2.yml +++ b/.github/workflows/benchmark_group_2.yml @@ -1,13 +1,11 @@ name: benchmark-group-2 on: [pull_request, push] + jobs: - start-runner: + start-benchmark-runner: name: Start self-hosted EC2 runner runs-on: ubuntu-latest outputs: - server-label: ${{ steps.start-server-ec2-runner.outputs.label }} - server-ec2-instance-id: ${{ steps.start-server-ec2-runner.outputs.ec2-instance-id }} - private-ip: ${{ steps.get-private-ip.outputs.private-ip }} benchmark-label: ${{ steps.start-benchmark-ec2-runner.outputs.label }} benchmark-ec2-instance-id: ${{ steps.start-benchmark-ec2-runner.outputs.ec2-instance-id }} steps: @@ -17,8 +15,8 @@ jobs: aws-access-key-id: ${{ secrets.GROUP_2_AWS_ACCESS_KEY_ID }} aws-secret-access-key: ${{ secrets.GROUP_2_AWS_SECRET_ACCESS_KEY }} aws-region: ${{ vars.AWS_REGION }} - - name: Start Server EC2 runner - id: start-server-ec2-runner + - name: Start Benchmark EC2 runner + id: start-benchmark-ec2-runner uses: machulav/ec2-github-runner@v2 with: mode: start @@ -27,8 +25,22 @@ jobs: ec2-instance-type: c5.xlarge subnet-id: subnet-0f0b102aeba4b75f7 security-group-id: sg-01dbe69e3eaf89995 - - name: Start Benchmark EC2 runner - id: start-benchmark-ec2-runner + start-server-runner: + name: Start self-hosted EC2 runner + runs-on: ubuntu-latest + outputs: + server-label: ${{ steps.start-server-ec2-runner.outputs.label }} + server-ec2-instance-id: ${{ steps.start-server-ec2-runner.outputs.ec2-instance-id }} + private-ip: ${{ steps.get-private-ip.outputs.private-ip }} + steps: + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v1 + with: + aws-access-key-id: ${{ secrets.GROUP_2_AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.GROUP_2_AWS_SECRET_ACCESS_KEY }} + aws-region: ${{ vars.AWS_REGION }} + - name: Start Server EC2 runner + id: start-server-ec2-runner uses: machulav/ec2-github-runner@v2 with: mode: start @@ -46,19 +58,19 @@ jobs: >> $GITHUB_OUTPUT run-server: name: Start the ISTZIIO server - needs: start-runner # required to start the main job when the runner is ready - runs-on: ${{ needs.start-runner.outputs.server-label }} # run the job on the newly created runner + needs: start-server-runner # required to start the main job when the runner is ready + runs-on: ${{ needs.start-server-runner.outputs.server-label }} # run the job on the newly created runner env: SERVER_ROOT: ${{ github.workspace }}/server AWS_ACCESS_KEY_ID: ${{ secrets.GROUP_2_AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.GROUP_2_AWS_SECRET_ACCESS_KEY }} - SERVER_IP: ${{ needs.start-runner.outputs.private-ip }} + SERVER_IP: ${{ needs.start-server-runner.outputs.private-ip }} steps: - name: Clone Server uses: actions/checkout@v4 with: repository: cmu-db/15721-s24-cache2 - ref: benchmark + ref: main - name: Build Server run: | cargo build --bin istziio_server_node @@ -70,13 +82,16 @@ jobs: cat ${{ github.workspace }}/logs/app_6379.log - name: spin run: sleep 60 + - name: Server shut down and log + run: cat ${{ github.workspace }}/logs/app_6379.log run-benchmark: name: Run Benchmark needs: - - start-runner - runs-on: ${{ needs.start-runner.outputs.benchmark-label }} # run the job on the newly created runner + - start-server-runner + - start-benchmark-runner + runs-on: ${{ needs.start-benchmark-runner.outputs.benchmark-label }} # run the job on the newly created runner env: - SERVER_URL: http://${{ needs.start-runner.outputs.private-ip }}:26379 + SERVER_URL: http://${{ needs.start-server-runner.outputs.private-ip }}:26379 steps: - name: Clone Server uses: actions/checkout@v4 @@ -93,7 +108,8 @@ jobs: stop-runner: name: Stop self-hosted EC2 runner needs: - - start-runner # required to get output from the start-runner job + - start-server-runner + - start-benchmark-runner # required to get output from the start-server-runner job - run-server # required to wait when the main job is done - run-benchmark runs-on: ubuntu-latest @@ -110,13 +126,13 @@ jobs: with: mode: stop github-token: ${{ secrets.GROUP_2_GH_PERSONAL_ACCESS_TOKEN }} - label: ${{ needs.start-runner.outputs.server-label }} - ec2-instance-id: ${{ needs.start-runner.outputs.server-ec2-instance-id }} + label: ${{ needs.start-server-runner.outputs.server-label }} + ec2-instance-id: ${{ needs.start-server-runner.outputs.server-ec2-instance-id }} - name: Stop Benchmark EC2 runner uses: machulav/ec2-github-runner@v2 with: mode: stop github-token: ${{ secrets.GROUP_2_GH_PERSONAL_ACCESS_TOKEN }} - label: ${{ needs.start-runner.outputs.benchmark-label }} - ec2-instance-id: ${{ needs.start-runner.outputs.benchmark-ec2-instance-id }} + label: ${{ needs.start-benchmark-runner.outputs.benchmark-label }} + ec2-instance-id: ${{ needs.start-benchmark-runner.outputs.benchmark-ec2-instance-id }} \ No newline at end of file diff --git a/.github/workflows/benchmark_group_2_local_client.yml b/.github/workflows/benchmark_group_2_local_client.yml new file mode 100644 index 0000000..caff477 --- /dev/null +++ b/.github/workflows/benchmark_group_2_local_client.yml @@ -0,0 +1,97 @@ +name: benchmark-group-2-local-client +on: [pull_request, push] + +jobs: + start-runner: + name: Start self-hosted EC2 runner + runs-on: ubuntu-latest + outputs: + server-label: ${{ steps.start-server-ec2-runner.outputs.label }} + server-ec2-instance-id: ${{ steps.start-server-ec2-runner.outputs.ec2-instance-id }} + private-ip: ${{ steps.get-private-ip.outputs.private-ip }} + steps: + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v1 + with: + aws-access-key-id: ${{ secrets.GROUP_2_AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.GROUP_2_AWS_SECRET_ACCESS_KEY }} + aws-region: ${{ vars.AWS_REGION }} + - name: Start Server EC2 runner + id: start-server-ec2-runner + uses: machulav/ec2-github-runner@v2 + with: + mode: start + github-token: ${{ secrets.GROUP_2_GH_PERSONAL_ACCESS_TOKEN }} + ec2-image-id: ami-0a1fee7d1926e55ce + ec2-instance-type: c5.xlarge + subnet-id: subnet-0f0b102aeba4b75f7 + security-group-id: sg-01dbe69e3eaf89995 + - name: Get private IP address + id: get-private-ip + run: | + echo "private-ip=$(aws ec2 describe-instances \ + --instance-id ${{ steps.start-server-ec2-runner.outputs.ec2-instance-id }} \ + --query 'Reservations[0].Instances[0].PrivateIpAddress' | tr -d '"')" \ + >> $GITHUB_OUTPUT + run-server: + name: Start the ISTZIIO server + needs: start-runner # required to start the main job when the runner is ready + runs-on: ${{ needs.start-runner.outputs.server-label }} # run the job on the newly created runner + env: + SERVER_ROOT: ${{ github.workspace }}/istziio/server + AWS_ACCESS_KEY_ID: ${{ secrets.GROUP_2_AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.GROUP_2_AWS_SECRET_ACCESS_KEY }} + SERVER_IP: 127.0.0.1 + SERVER_URL: http://127.0.0.1:26379 + steps: + - name: Clone Server + uses: actions/checkout@v4 + with: + path: 'istziio' + repository: cmu-db/15721-s24-cache2 + ref: main + - name: Build Server + working-directory: ${{ github.workspace }}/istziio + run: | + cargo build --bin istziio_server_node + - name: Clone benchmark + uses: actions/checkout@v4 + with: + path: 'benchmark' + - name: Build Benchmark driver + working-directory: ${{ github.workspace }}/benchmark + run: cargo build + - name: start server nodes + working-directory: ${{ github.workspace }}/istziio + run: server/run.sh + - name: wait for server completing setups + run: | + sleep 5 + cat ${{ github.workspace }}/istziio/logs/app_6379.log + - name: Run Benchmark + working-directory: ${{ github.workspace }}/benchmark + run: cargo run + - name: Server shut down and log + run: cat ${{ github.workspace }}/istziio/logs/app_6379.log + stop-runner: + name: Stop self-hosted EC2 runner + needs: + - start-runner # required to get output from the start-runner job + - run-server # required to wait when the main job is done + runs-on: ubuntu-latest + if: ${{ always() }} # required to stop the runner even if the error happened in the previous jobs + steps: + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v1 + with: + aws-access-key-id: ${{ secrets.GROUP_2_AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.GROUP_2_AWS_SECRET_ACCESS_KEY }} + aws-region: ${{ vars.AWS_REGION }} + - name: Stop Server EC2 runner + uses: machulav/ec2-github-runner@v2 + with: + mode: stop + github-token: ${{ secrets.GROUP_2_GH_PERSONAL_ACCESS_TOKEN }} + label: ${{ needs.start-runner.outputs.server-label }} + ec2-instance-id: ${{ needs.start-runner.outputs.server-ec2-instance-id }} + \ No newline at end of file diff --git a/.github/workflows/benchmark_group_2_multi.yml b/.github/workflows/benchmark_group_2_multi.yml new file mode 100644 index 0000000..ac275fb --- /dev/null +++ b/.github/workflows/benchmark_group_2_multi.yml @@ -0,0 +1,308 @@ +name: benchmark-group-2-multi +on: [pull_request, push] +jobs: + start-benchmark-runner: + name: Start self-hosted EC2 runner + runs-on: ubuntu-latest + outputs: + benchmark-label: ${{ steps.start-benchmark-ec2-runner.outputs.label }} + benchmark-ec2-instance-id: ${{ steps.start-benchmark-ec2-runner.outputs.ec2-instance-id }} + steps: + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v1 + with: + aws-access-key-id: ${{ secrets.GROUP_2_AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.GROUP_2_AWS_SECRET_ACCESS_KEY }} + aws-region: ${{ vars.AWS_REGION }} + - name: Start Benchmark EC2 runner + id: start-benchmark-ec2-runner + uses: machulav/ec2-github-runner@v2 + with: + mode: start + github-token: ${{ secrets.GROUP_2_GH_PERSONAL_ACCESS_TOKEN }} + ec2-image-id: ami-0a1fee7d1926e55ce + ec2-instance-type: c5.xlarge + subnet-id: subnet-0f0b102aeba4b75f7 + security-group-id: sg-01dbe69e3eaf89995 + start-server-1-runner: + name: Start self-hosted EC2 runner + runs-on: ubuntu-latest + outputs: + server-label: ${{ steps.start-server-ec2-runner.outputs.label }} + server-ec2-instance-id: ${{ steps.start-server-ec2-runner.outputs.ec2-instance-id }} + private-ip: ${{ steps.get-private-ip.outputs.private-ip }} + steps: + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v1 + with: + aws-access-key-id: ${{ secrets.GROUP_2_AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.GROUP_2_AWS_SECRET_ACCESS_KEY }} + aws-region: ${{ vars.AWS_REGION }} + - name: Start Server EC2 runner + id: start-server-ec2-runner + uses: machulav/ec2-github-runner@v2 + with: + mode: start + github-token: ${{ secrets.GROUP_2_GH_PERSONAL_ACCESS_TOKEN }} + ec2-image-id: ami-0a1fee7d1926e55ce + ec2-instance-type: c5.xlarge + subnet-id: subnet-0f0b102aeba4b75f7 + security-group-id: sg-01dbe69e3eaf89995 + - name: Get private IP address + id: get-private-ip + run: | + echo "private-ip=$(aws ec2 describe-instances \ + --instance-id ${{ steps.start-server-ec2-runner.outputs.ec2-instance-id }} \ + --query 'Reservations[0].Instances[0].PrivateIpAddress' | tr -d '"')" \ + >> $GITHUB_OUTPUT + start-server-2-runner: + name: Start self-hosted EC2 runner + runs-on: ubuntu-latest + outputs: + server-label: ${{ steps.start-server-ec2-runner.outputs.label }} + server-ec2-instance-id: ${{ steps.start-server-ec2-runner.outputs.ec2-instance-id }} + private-ip: ${{ steps.get-private-ip.outputs.private-ip }} + steps: + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v1 + with: + aws-access-key-id: ${{ secrets.GROUP_2_AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.GROUP_2_AWS_SECRET_ACCESS_KEY }} + aws-region: ${{ vars.AWS_REGION }} + - name: Start Server EC2 runner + id: start-server-ec2-runner + uses: machulav/ec2-github-runner@v2 + with: + mode: start + github-token: ${{ secrets.GROUP_2_GH_PERSONAL_ACCESS_TOKEN }} + ec2-image-id: ami-0a1fee7d1926e55ce + ec2-instance-type: c5.xlarge + subnet-id: subnet-0f0b102aeba4b75f7 + security-group-id: sg-01dbe69e3eaf89995 + - name: Get private IP address + id: get-private-ip + run: | + echo "private-ip=$(aws ec2 describe-instances \ + --instance-id ${{ steps.start-server-ec2-runner.outputs.ec2-instance-id }} \ + --query 'Reservations[0].Instances[0].PrivateIpAddress' | tr -d '"')" \ + >> $GITHUB_OUTPUT + start-server-3-runner: + name: Start self-hosted EC2 runner + runs-on: ubuntu-latest + outputs: + server-label: ${{ steps.start-server-ec2-runner.outputs.label }} + server-ec2-instance-id: ${{ steps.start-server-ec2-runner.outputs.ec2-instance-id }} + private-ip: ${{ steps.get-private-ip.outputs.private-ip }} + steps: + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v1 + with: + aws-access-key-id: ${{ secrets.GROUP_2_AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.GROUP_2_AWS_SECRET_ACCESS_KEY }} + aws-region: ${{ vars.AWS_REGION }} + - name: Start Server EC2 runner + id: start-server-ec2-runner + uses: machulav/ec2-github-runner@v2 + with: + mode: start + github-token: ${{ secrets.GROUP_2_GH_PERSONAL_ACCESS_TOKEN }} + ec2-image-id: ami-0a1fee7d1926e55ce + ec2-instance-type: c5.xlarge + subnet-id: subnet-0f0b102aeba4b75f7 + security-group-id: sg-01dbe69e3eaf89995 + - name: Get private IP address + id: get-private-ip + run: | + echo "private-ip=$(aws ec2 describe-instances \ + --instance-id ${{ steps.start-server-ec2-runner.outputs.ec2-instance-id }} \ + --query 'Reservations[0].Instances[0].PrivateIpAddress' | tr -d '"')" \ + >> $GITHUB_OUTPUT + run-server-1: + name: Start the ISTZIIO server 1 + needs: + - start-server-1-runner + - start-server-2-runner + - start-server-3-runner # required to start the main job when the runner is ready + runs-on: ${{ needs.start-server-1-runner.outputs.server-label }} # run the job on the newly created runner + env: + SERVER_ROOT: ${{ github.workspace }}/server + AWS_ACCESS_KEY_ID: ${{ secrets.GROUP_2_AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.GROUP_2_AWS_SECRET_ACCESS_KEY }} + SERVER_IP: ${{ needs.start-server-1-runner.outputs.private-ip }} + NODE_1_IP: ${{ needs.start-server-1-runner.outputs.private-ip }} + NODE_2_IP: ${{ needs.start-server-2-runner.outputs.private-ip }} + NODE_3_IP: ${{ needs.start-server-3-runner.outputs.private-ip }} + steps: + - name: Clone Server + uses: actions/checkout@v4 + with: + repository: cmu-db/15721-s24-cache2 + ref: main + - name: Build Server + run: | + cargo build --bin istziio_server_node + - name: start server nodes + run: ${{ github.workspace }}/server/run_single_node.sh + - name: wait for server completing setups + run: | + sleep 5 + cat ${{ github.workspace }}/logs/app_6379.log + - name: spin + run: sleep 60 + - name: Server shut down and log + run: cat ${{ github.workspace }}/logs/app_6379.log + run-server-2: + name: Start the ISTZIIO server 2 + needs: + - start-server-1-runner + - start-server-2-runner + - start-server-3-runner # required to start the main job when the runner is ready + runs-on: ${{ needs.start-server-2-runner.outputs.server-label }} # run the job on the newly created runner + env: + SERVER_ROOT: ${{ github.workspace }}/server + AWS_ACCESS_KEY_ID: ${{ secrets.GROUP_2_AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.GROUP_2_AWS_SECRET_ACCESS_KEY }} + SERVER_IP: ${{ needs.start-server-2-runner.outputs.private-ip }} + NODE_1_IP: ${{ needs.start-server-1-runner.outputs.private-ip }} + NODE_2_IP: ${{ needs.start-server-2-runner.outputs.private-ip }} + NODE_3_IP: ${{ needs.start-server-3-runner.outputs.private-ip }} + steps: + - name: Clone Server + uses: actions/checkout@v4 + with: + repository: cmu-db/15721-s24-cache2 + ref: main + - name: Build Server + run: | + cargo build --bin istziio_server_node + - name: start server nodes + run: ${{ github.workspace }}/server/run_single_node.sh + - name: wait for server completing setups + run: | + sleep 5 + cat ${{ github.workspace }}/logs/app_6379.log + - name: spin + run: sleep 60 + - name: Server shut down and log + run: cat ${{ github.workspace }}/logs/app_6379.log + run-server-3: + name: Start the ISTZIIO server 3 + needs: + - start-server-1-runner + - start-server-2-runner + - start-server-3-runner # required to start the main job when the runner is ready + runs-on: ${{ needs.start-server-3-runner.outputs.server-label }} # run the job on the newly created runner + env: + SERVER_ROOT: ${{ github.workspace }}/server + AWS_ACCESS_KEY_ID: ${{ secrets.GROUP_2_AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.GROUP_2_AWS_SECRET_ACCESS_KEY }} + SERVER_IP: ${{ needs.start-server-3-runner.outputs.private-ip }} + NODE_1_IP: ${{ needs.start-server-1-runner.outputs.private-ip }} + NODE_2_IP: ${{ needs.start-server-2-runner.outputs.private-ip }} + NODE_3_IP: ${{ needs.start-server-3-runner.outputs.private-ip }} + steps: + - name: Clone Server + uses: actions/checkout@v4 + with: + repository: cmu-db/15721-s24-cache2 + ref: main + - name: Build Server + run: | + cargo build --bin istziio_server_node + - name: start server nodes + run: ${{ github.workspace }}/server/run_single_node.sh -c + - name: wait for server completing setups + run: | + sleep 5 + cat ${{ github.workspace }}/logs/app_6379.log + - name: spin + run: sleep 60 + - name: Server shut down and log + run: cat ${{ github.workspace }}/logs/app_6379.log + run-benchmark: + name: Run Benchmark + needs: + - start-server-1-runner + - start-server-2-runner + - start-server-3-runner + - start-benchmark-runner + runs-on: ${{ needs.start-benchmark-runner.outputs.benchmark-label }} # run the job on the newly created runner + env: + SERVER_URL: http://${{ needs.start-server-1-runner.outputs.private-ip }}:26379 + SERVER_1_URL: http://${{ needs.start-server-1-runner.outputs.private-ip }}:26379 + SERVER_2_URL: http://${{ needs.start-server-2-runner.outputs.private-ip }}:26379 + SERVER_3_URL: http://${{ needs.start-server-3-runner.outputs.private-ip }}:26379 + steps: + - name: Clone Server + uses: actions/checkout@v4 + - name: Build Benchmark driver + run: cargo build + - name: Wait for server 1 + run: | + until curl --output /dev/null --silent --head --fail $SERVER_1_URL/; do + printf '.' + sleep 5 + done + - name: Wait for server 2 + run: | + until curl --output /dev/null --silent --head --fail $SERVER_2_URL/; do + printf '.' + sleep 5 + done + - name: Wait for server 3 + run: | + until curl --output /dev/null --silent --head --fail $SERVER_3_URL/; do + printf '.' + sleep 5 + done + - name: Run Benchmark + run: cargo run + stop-runner: + name: Stop self-hosted EC2 runner + needs: + - start-benchmark-runner + - start-server-1-runner + - start-server-2-runner + - start-server-3-runner # required to get output from the start-runner job + - run-server-1 + - run-server-2 + - run-server-3 + - run-benchmark + runs-on: ubuntu-latest + if: ${{ always() }} # required to stop the runner even if the error happened in the previous jobs + steps: + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v1 + with: + aws-access-key-id: ${{ secrets.GROUP_2_AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.GROUP_2_AWS_SECRET_ACCESS_KEY }} + aws-region: ${{ vars.AWS_REGION }} + - name: Stop Server 1 EC2 runner + uses: machulav/ec2-github-runner@v2 + with: + mode: stop + github-token: ${{ secrets.GROUP_2_GH_PERSONAL_ACCESS_TOKEN }} + label: ${{ needs.start-server-1-runner.outputs.server-label }} + ec2-instance-id: ${{ needs.start-server-1-runner.outputs.server-ec2-instance-id }} + - name: Stop Server 2 EC2 runner + uses: machulav/ec2-github-runner@v2 + with: + mode: stop + github-token: ${{ secrets.GROUP_2_GH_PERSONAL_ACCESS_TOKEN }} + label: ${{ needs.start-server-2-runner.outputs.server-label }} + ec2-instance-id: ${{ needs.start-server-2-runner.outputs.server-ec2-instance-id }} + - name: Stop Server 3 EC2 runner + uses: machulav/ec2-github-runner@v2 + with: + mode: stop + github-token: ${{ secrets.GROUP_2_GH_PERSONAL_ACCESS_TOKEN }} + label: ${{ needs.start-server-3-runner.outputs.server-label }} + ec2-instance-id: ${{ needs.start-server-3-runner.outputs.server-ec2-instance-id }} + - name: Stop Benchmark EC2 runner + uses: machulav/ec2-github-runner@v2 + with: + mode: stop + github-token: ${{ secrets.GROUP_2_GH_PERSONAL_ACCESS_TOKEN }} + label: ${{ needs.start-benchmark-runner.outputs.benchmark-label }} + ec2-instance-id: ${{ needs.start-benchmark-runner.outputs.benchmark-ec2-instance-id }} + \ No newline at end of file diff --git a/pqt_file_list.txt b/pqt_file_list.txt new file mode 100644 index 0000000..6e7eb21 --- /dev/null +++ b/pqt_file_list.txt @@ -0,0 +1,30 @@ +1mb/random_data_1m_0.parquet +1mb/random_data_1m_1.parquet +1mb/random_data_1m_2.parquet +1mb/random_data_1m_3.parquet +1mb/random_data_1m_4.parquet +1mb/random_data_1m_5.parquet +1mb/random_data_1m_6.parquet +1mb/random_data_1m_7.parquet +1mb/random_data_1m_8.parquet +10mb/random_data_1m_9.parquet +10mb/random_data_10m_0.parquet +10mb/random_data_10m_1.parquet +10mb/random_data_10m_2.parquet +10mb/random_data_10m_3.parquet +10mb/random_data_10m_4.parquet +10mb/random_data_10m_5.parquet +10mb/random_data_10m_6.parquet +10mb/random_data_10m_7.parquet +10mb/random_data_10m_8.parquet +10mb/random_data_10m_9.parquet +100mb/random_data_100m_0.parquet +100mb/random_data_100m_1.parquet +100mb/random_data_100m_2.parquet +100mb/random_data_100m_3.parquet +100mb/random_data_100m_4.parquet +100mb/random_data_100m_5.parquet +100mb/random_data_100m_6.parquet +100mb/random_data_100m_7.parquet +100mb/random_data_100m_8.parquet +100mb/random_data_100m_9.parquet \ No newline at end of file diff --git a/pqt_file_list_clean.txt b/pqt_file_list_clean.txt new file mode 100644 index 0000000..28e9283 --- /dev/null +++ b/pqt_file_list_clean.txt @@ -0,0 +1,20 @@ +random_data_1m_0.parquet +random_data_1m_1.parquet +random_data_1m_2.parquet +random_data_1m_3.parquet +random_data_1m_4.parquet +random_data_1m_5.parquet +random_data_1m_6.parquet +random_data_1m_7.parquet +random_data_1m_8.parquet +random_data_1m_9.parquet +random_data_100m_0.parquet +random_data_100m_1.parquet +random_data_100m_2.parquet +random_data_100m_3.parquet +random_data_100m_4.parquet +random_data_100m_5.parquet +random_data_100m_6.parquet +random_data_100m_7.parquet +random_data_100m_8.parquet +random_data_100m_9.parquet \ No newline at end of file diff --git a/src/main.rs b/src/main.rs index 329aaab..f02082f 100644 --- a/src/main.rs +++ b/src/main.rs @@ -5,6 +5,8 @@ use std::path::PathBuf; async fn main() { // benchmark_sync().await; // benchmark_parallel().await; - let trace = parse_trace(PathBuf::from("traces/trace1.csv")).unwrap(); - run_trace(trace, &utils::setup_client_1).await; + let trace = parse_trace(PathBuf::from("traces/trace_1m.csv")).unwrap(); + run_trace(trace, &utils::setup_client_2).await; + let trace = parse_trace(PathBuf::from("traces/trace_100m.csv")).unwrap(); + run_trace(trace, &utils::setup_client_2).await; } \ No newline at end of file diff --git a/src/utils.rs b/src/utils.rs index cdbaddd..b2e9d05 100644 --- a/src/utils.rs +++ b/src/utils.rs @@ -1,6 +1,7 @@ use istziio_client::client_api::{StorageClient, TableId}; use istziio_client::storage_client::StorageClientImpl; use std::collections::HashMap; +use std::io::BufRead; pub fn setup_client_1() -> Box { let server_url = std::env::var("SERVER_URL").unwrap_or(String::from("http://127.0.0.1:26379")); @@ -12,7 +13,7 @@ pub fn setup_client_1() -> Box { pub fn setup_client_2() -> Box { let server_url = std::env::var("SERVER_URL").unwrap_or(String::from("http://127.0.0.1:26379")); println!("server url: {}", &server_url); - let map = create_table_file_map().unwrap(); + let map = create_table_file_map_from_list().unwrap(); Box::new(StorageClientImpl::new_for_test(1, map.clone(), &server_url)) } @@ -26,3 +27,16 @@ pub fn create_table_file_map() -> Result, std::io::Erro Ok(table_file_map) } + +pub fn create_table_file_map_from_list() -> Result, std::io::Error> { + let mut table_file_map: HashMap = HashMap::new(); + // Scan file pqt_file_list.txt line by line, each line is a file + let file = std::fs::File::open("pqt_file_list_clean.txt")?; + let reader = std::io::BufReader::new(file); + for line in reader.lines() { + let line = line.unwrap(); + let id = table_file_map.len() as TableId; + table_file_map.insert(id, line); + } + Ok(table_file_map) +} diff --git a/traces/trace_1m.csv b/traces/trace_1m.csv index 12c4f92..82acc48 100644 --- a/traces/trace_1m.csv +++ b/traces/trace_1m.csv @@ -1,21 +1,21 @@ -timestamp,file_index -0,1 -188,2 -529,1 -968,8 -1333,2 -1767,2 -1781,1 -2053,1 -2072,3 -2168,5 -2596,1 -3039,2 -3216,3 -3649,3 -3669,2 -3977,2 -4188,2 -4564,8 -4740,2 -5093,1 +timestamp,file_index +0,1 +188,2 +529,1 +968,8 +1333,2 +1767,2 +1781,1 +2053,1 +2072,3 +2168,5 +2596,1 +3039,2 +3216,3 +3649,3 +3669,2 +3977,2 +4188,2 +4564,8 +4740,2 +5093,1