Merge pull request #2262 from GoogleCloudPlatform/ospo-compliance

chore: add license header and CONTRIBUTING.md

CONTRIBUTING.md

@@ -0,0 +1,32 @@
+# How to Contribute
+
+We would love to accept your patches and contributions to this project.
+
+## Before you begin
+
+### Sign our Contributor License Agreement
+
+Contributions to this project must be accompanied by a
+[Contributor License Agreement](https://cla.developers.google.com/about) (CLA).
+You (or your employer) retain the copyright to your contribution; this simply
+gives us permission to use and redistribute your contributions as part of the
+project.
+
+If you or your current employer have already signed the Google CLA (even if it
+was for a different project), you probably don't need to do it again.
+
+Visit <https://cla.developers.google.com/> to see your current agreements or to
+sign a new one.
+
+### Review our Community Guidelines
+
+This project follows [Google's Open Source Community
+Guidelines](https://opensource.google/conduct/).
+
+## Contribution process
+
+### Code Reviews
+
+All submissions, including submissions by project members, require review. We
+use [GitHub pull requests](https://docs.github.com/articles/about-pull-requests)
+for this purpose.

internal/gcsx/append_object_creator.go

@@ -1,4 +1,4 @@
-// Copyright 2015 Google Inc. All Rights Reserved.
+// Copyright 2024 Google Inc. All Rights Reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.

internal/gcsx/bucket_manager_test.go

@@ -1,3 +1,17 @@
+// Copyright 2015 Google Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package gcsx
 
 import (

perfmetrics/scripts/compare_fuse_types_using_fio.py

@@ -1,13 +1,27 @@
+# Copyright 2024 Google Inc. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 """Python script to do FIO test for GCSFuse Vs GCSFuse/Goofys.
 
 To run the script:
-python3 compare_fuse_types_using_fio.py -- --fuse_type_1=<value1> --fuse_type_1_version=<version1> --flags_1=<flags1> 
+python3 compare_fuse_types_using_fio.py -- --fuse_type_1=<value1> --fuse_type_1_version=<version1> --flags_1=<flags1>
 --fuse_type_2=<value2> --fuse_type_2_version=<version2> --flags_2=<flags2> --jobfile_path=<jobfile_path> --gcs_bucket=<gcs_bucket>
 
 1) --fuse_type_1=<value1> & --fuse_type_2=<value2>
 -> Value1 and value2 can be fuse based filesystem name.
--> fuse based filesystem name is used for logging so it can be anything. 
--> For GCSFuse it should be 'gcsfuse' 
+-> fuse based filesystem name is used for logging so it can be anything.
+-> For GCSFuse it should be 'gcsfuse'
 2) --fuse_type_1_version=<version1> & --fuse_type_2_version=<version2>
 -> For GCSFuse you can mention any released version or ‘master’ as version if you want to build from source.
 -> For other fuse based filesystem provide the link of github repository.
@@ -20,8 +34,8 @@
 5) --gcs_bucket=<gcs_bucket>
 -> Name of the GCS bucket to be mounted for FIO load test.
 
-The code takes input the fuse based filesystem type and its version, required flags to mount the bucket, jobfile_path, 
-and gcs bucket name as input for fio test. Then it will perform the fio test for each of the given 
+The code takes input the fuse based filesystem type and its version, required flags to mount the bucket, jobfile_path,
+and gcs bucket name as input for fio test. Then it will perform the fio test for each of the given
 fuse based filesystem type at a specific version
 and extract fio_metrics to 'output.txt' file in the 'out' directory.
 """
@@ -38,9 +52,9 @@
 
 def _install_gcsfuse(version, gcs_bucket, gcsfuse_flags) -> None:
   """Install gcsfuse with Specific version.
-  
+
   Args:
-    version(str): Gcsfuse version to be installed. 
+    version(str): Gcsfuse version to be installed.
     gcs_bucket(str): GCS bucket to be mounted.
     gcsfuse_flags(str): Fuse flags for mounting the GCS bucket.
   """
@@ -53,9 +67,9 @@ def _install_gcsfuse(version, gcs_bucket, gcsfuse_flags) -> None:
 
 def _install_gcsfuse_source(gcs_bucket, gcsfuse_flags) -> None:
   """Run gcsfuse from source code.
-  
+
   Args:
-    version(str): Gcsfuse version to be installed. 
+    version(str): Gcsfuse version to be installed.
     gcs_bucket(str): GCS bucket to be mounted.
   """
   os.system(f'''git clone {GCSFUSE_REPO}
@@ -65,10 +79,10 @@ def _install_gcsfuse_source(gcs_bucket, gcsfuse_flags) -> None:
             cd ..
             ''')
 
-  
+
 def _remove_gcsfuse(version) -> None:
   """Remove gcsfuse with specific version.
-  
+
   Args:
     version(str): Gcsfuse version to be removed.
   """
@@ -82,7 +96,7 @@ def _remove_gcsfuse(version) -> None:
 
 def _install_fuse(fuse_type, fuse_url, fuse_flags, gcs_bucket) -> None:
   """Install latest version of given fuse based filesystem.
-  
+
   Args:
     fuse_type(str): Fuse type for fio test.
     fuse_url(str): URL of github repo of given fuse based filesystem.
@@ -100,7 +114,7 @@ def _install_fuse(fuse_type, fuse_url, fuse_flags, gcs_bucket) -> None:
 
 def _remove_fuse(fuse_type) -> None:
   """Remove the given fuse based filesystem.
-  
+
   Args:
     fuse_type(str): Fuse type for fio test.
   """
@@ -112,7 +126,7 @@ def _remove_fuse(fuse_type) -> None:
 
 def _run_fio_test(jobfile_path, fio_metrics_obj) -> None:
   """Run fio test and extract metrics to output.txt file.
-  
+
   Args:
     jobfile(str): Path of the job file.
     fio_metrics_obj(str): Object for extracting fio metrics.
@@ -127,7 +141,7 @@ def _run_fio_test(jobfile_path, fio_metrics_obj) -> None:
 
 def _gcsfuse_fio_test(version, jobfile_path, fio_metrics_obj, gcs_bucket, fuse_flags) -> None:
   """FIO test for gcsfuse of given version.
-  
+
   Args:
     version(str): Gcsfuse version to perform fio test.
     jobfile(str): Path of the job file.
@@ -139,14 +153,14 @@ def _gcsfuse_fio_test(version, jobfile_path, fio_metrics_obj, gcs_bucket, fuse_f
     _install_gcsfuse_source(gcs_bucket, fuse_flags)
   else:
     _install_gcsfuse(version, gcs_bucket, fuse_flags)
-  
+
   _run_fio_test(jobfile_path, fio_metrics_obj)
   _remove_gcsfuse(version)
 
 
 def _fuse_fio_test(fuse_type, jobfile_path, fio_metrics_obj, gcs_bucket, fuse_flags, fuse_url) -> None:
   """FIO test for latest version of goofys.
-  
+
   Args:
     fuse_type(str): Fuse type for fio test.
     jobfile(str): Path of the job file.
@@ -159,10 +173,10 @@ def _fuse_fio_test(fuse_type, jobfile_path, fio_metrics_obj, gcs_bucket, fuse_fl
   _run_fio_test(jobfile_path, fio_metrics_obj)
   _remove_fuse(fuse_type)
 
-  
+
 def _fuse_test(fuse_type, fuse_type_version, jobfile_path, fio_metric_obj, gcs_bucket, fuse_flags) -> None:
   """FIO test for specific version of given fuse type.
-  
+
   Args:
     fuse_type(str): Fuse type for fio test.
     fuse_type_version(str): Fuse type version for fio test.
@@ -175,8 +189,8 @@ def _fuse_test(fuse_type, fuse_type_version, jobfile_path, fio_metric_obj, gcs_b
     _gcsfuse_fio_test(fuse_type_version, jobfile_path, fio_metric_obj, gcs_bucket, fuse_flags)
   else:
     _fuse_fio_test(fuse_type, jobfile_path, fio_metric_obj, gcs_bucket, fuse_flags, fuse_type_version)
-      
-  
+
+
 def main(argv) -> None:
 
   parser = argparse.ArgumentParser()
@@ -226,4 +240,4 @@ def main(argv) -> None:
 
 if __name__ == '__main__':
   app.run(main)
-  
+

perfmetrics/scripts/continuous_test/ml_tests/tf/resnet/build.sh

@@ -1,4 +1,17 @@
 #!/bin/bash
+# Copyright 2024 Google Inc. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
 
 # This will stop execution when any command will have non-zero status.
 set -e

perfmetrics/scripts/fio/constants.py

@@ -1,3 +1,17 @@
+# Copyright 2023 Google Inc. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 """Contains constants for fio_metrics.
 """
 

perfmetrics/scripts/fio/fio_metrics_test.py

@@ -1,3 +1,17 @@
+# Copyright 2024 Google Inc. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 """Tests for fio_metrics.
 
   Usage from perfmetrics/scripts folder: python3 -m fio.fio_metrics_test

perfmetrics/scripts/generate_files.py

@@ -1,3 +1,17 @@
+# Copyright 2024 Google Inc. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 """This script takes input the number of files and the file size and creates
 required files and simultaneously uploads them on the gcs bucket.
 The progress is updated in 'output.out' file.

perfmetrics/scripts/gsheet/gsheet.py

@@ -1,3 +1,17 @@
+# Copyright 2024 Google Inc. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 from google.oauth2 import service_account
 from googleapiclient.discovery import build
 

perfmetrics/scripts/gsheet/gsheet_test.py

@@ -1,3 +1,17 @@
+# Copyright 2024 Google Inc. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 """Tests for gsheet."""
 import unittest
 from unittest import mock

perfmetrics/scripts/ls_metrics/directory.proto

@@ -1,3 +1,17 @@
+// Copyright 2024 Google Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 syntax = "proto3";
 
 package perfmetrics;

perfmetrics/scripts/ls_metrics/run_ls_benchmark.sh

@@ -1,4 +1,18 @@
 #!/bin/bash
+# Copyright 2024 Google Inc. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 set -e
 # Refer for env var: https://stackoverflow.com/questions/72441758/typeerror-descriptors-cannot-not-be-created-directly
 export PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=python

perfmetrics/scripts/ml_tests/setup_host.sh

@@ -1,4 +1,18 @@
 #!/bin/bash
+# Copyright 2024 Google Inc. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 # This file installs docker engine and nvidia driver and nvidia container tool
 # necessary for running dlc container on the vm