Pods accounting.

deic-dk · Dec 11, 2024 · e60ff76 · e60ff76
1 parent 15eff37
commit e60ff76
Show file tree

Hide file tree

Showing 3 changed files with 118 additions and 44 deletions.
diff --git a/lib/backgroundjob/stats.php b/lib/backgroundjob/stats.php
@@ -41,7 +41,17 @@ protected function run($argument) {
 	}
 
 	private function updateAndBill() {
-		$users = \OC_User::getUsers();
+		// If dryrunbillingusers is set in config.php, only calculate use for these,
+		// don't write to the DB and write bills with prefix 'test-'.
+		$dryrunusers = \OCP\Config::getSystemValue('dryrunbillingusers', '');
+		$dryrun = false;
+		if(!empty($dryrunusers)){
+			$users = $dryrunusers;
+			$dryrun = true;
+		}
+		else{
+			$users = \OC_User::getUsers();
+		}
 		foreach ($users as $user) {
 			// Add a line to usage-201*.txt locally.
 			// logDailyUsage checks if the line already exists and bails if so.
@@ -64,11 +74,15 @@ private function updateAndBill() {
 				\OC_Log::write('files_accounting',"Not billing on non-home server for ".$user, \OC_Log::WARN);
 				continue;
 			}
-			$this->updateAndBillUser($user);
+			$this->updateAndBillUser($user, $dryrun);
 		}
 	}
 
-	private function updateAndBillUser($user){
+	/**
+	 * @param string $user
+	 * @param boolean $dryrun If true, calculate charges, don't write to DB, write bill with prefix 'test-'
+	 */
+	private function updateAndBillUser($user, $dryrun=false){
 		if(!\OC_User::userExists($user)){
 			\OC_Log::write('files_accounting',"ERROR: Cannot bill non-existing user. ".$user, \OC_Log::ERROR);
 			return;
@@ -95,7 +109,7 @@ private function updateAndBillUser($user){
 		}
 
 		// Only run billing on the billing day
-		if(date("j", $this->timestamp) != $this->billingDay){
+		if(date("j", $this->timestamp) != $this->billingDay && !$dryrun){
 			\OCP\Util::writeLog('Files_Accounting', 'Not billing user: '.$user.' today', \OCP\Util::WARN);
 			return;
 		}
@@ -106,7 +120,7 @@ private function updateAndBillUser($user){
 		// A user who has a not expired  preapproval key is charged.
 		$hasPreapprovalKey = \OCA\Files_Accounting\Storage_Lib::getPreapprovalKey($user, $this->billingMonth,
 				$this->billingYear);
-		if($hasPreapprovalKey){
+		if($hasPreapprovalKey && !$dryrun){
 			ActivityHooks::automaticPaymentComplete($user,
 			array('month'=>$this->billingMonthName, 'year'=>$this->billingYear, 'item_number'=>$reference_id));
 		}
@@ -118,12 +132,12 @@ private function updateAndBillUser($user){
 		}
 		$files = scandir($path);
 		$currentMonthFiles = preg_grep("/^".$this->billingYear."-".$this->billingMonth."-.*\.pdf$/", $files);
-		if(!empty($currentMonthFiles)){
+		if(!empty($currentMonthFiles) && !$dryrun){
 			\OCP\Util::writeLog('Files_Accounting', 'Already billed user: '.$user.' for '.$this->billingMonthName, \OCP\Util::WARN);
 			return;
 		}
 
-		// Log monthly average to DB on master
+		// Log monthly average to DB on master if not dryrunning
 		$charge = \OCA\Files_Accounting\Storage_Lib::getChargeForUserServers($user);
 		$monthlyUsageAverage = \OCA\Files_Accounting\Storage_Lib::currentUsageAverage(
 				$user, $this->billingYear, $this->billingMonth, $this->timestamp);
@@ -173,7 +187,7 @@ private function updateAndBillUser($user){
 		else{
 			$totalSumDue = $sumDue;
 		}
-		if(isset($newPrePaid)){
+		if(isset($newPrePaid) && !$dryrun){
 			\OCA\Files_Accounting\Storage_Lib::setPrePaid($user, $newPrePaid);
 		}
 
@@ -224,34 +238,38 @@ private function updateAndBillUser($user){
 		}
 
 		// This goes to master
-		\OCA\Files_Accounting\Storage_Lib::updateMonth($user,
-				$hasPreapprovalKey||$totalSumDue==0?\OCA\Files_Accounting\Storage_Lib::PAYMENT_STATUS_PAID:
-				\OCA\Files_Accounting\Storage_Lib::PAYMENT_STATUS_PENDING,
-				$this->billingYear, $this->billingMonth, $this->timestamp, $this->dueTimestamp, $homeGB, $backupGB, $trashGB,
-				$charge['id_home'], $charge['id_backup'], $charge['url_home'], $charge['url_backup'], $charge['site_home'],
-				$charge['site_backup'], $totalSumDue, $reference_id);
-
+		if(!$dryrun){
+			\OCA\Files_Accounting\Storage_Lib::updateMonth($user,
+					$hasPreapprovalKey||$totalSumDue==0?\OCA\Files_Accounting\Storage_Lib::PAYMENT_STATUS_PAID:
+					\OCA\Files_Accounting\Storage_Lib::PAYMENT_STATUS_PENDING,
+					$this->billingYear, $this->billingMonth, $this->timestamp, $this->dueTimestamp, $homeGB, $backupGB, $trashGB,
+					$charge['id_home'], $charge['id_backup'], $charge['url_home'], $charge['url_backup'], $charge['site_home'],
+					$charge['site_backup'], $totalSumDue, $reference_id);
+		}
+
 		// Create invoice and store locally. It can always be recreated from DB on master (storage)
 		// and the files "files_accounting/pods/podsusage\_[year]\_[month].txt" on the home silo.
 		$filename = $this->invoice($user, $reference_id,
 				$homeGB+$trashGB, $backupGB, $totalSumDue,
 				$homeDue, $backupDue,
 				$charge['site_home'], $charge['site_backup'],
-				$groupUsagesGB, $groupCharges, $podsUse);
+				$groupUsagesGB, $groupCharges, $podsUse, $dryrun);
 
 		if(empty($filename)){
 			\OCP\Util::writeLog('Files_Accounting', 'ERROR: could not create invoice for '.$user.', '.$this->billingMonth.', '.$totalSumDue, \OCP\Util::ERROR);
 			return;
 		}
 
 		// Notify
-		ActivityHooks::invoiceCreate($user,
-			array('month'=>$this->billingMonthName, 'year'=>$this->billingYear, 'item_number'=>$reference_id,
-					'priority'=>($totalSumDue==0?\OCA\UserNotification\Data::PRIORITY_MEDIUM:\OCA\UserNotification\Data::PRIORITY_VERYHIGH)
-			));
-
-		// TODO: uncomment when this goes into production
-		//$this->sendNotificationMail($user, $totalSumDue, $filename, $charge['site_home']);
+		if(!$dryrun){
+			ActivityHooks::invoiceCreate($user,
+					array('month'=>$this->billingMonthName, 'year'=>$this->billingYear, 'item_number'=>$reference_id,
+							'priority'=>($totalSumDue==0?\OCA\UserNotification\Data::PRIORITY_MEDIUM:\OCA\UserNotification\Data::PRIORITY_VERYHIGH)
+					));
+
+			// TODO: uncomment when this goes into production
+			//$this->sendNotificationMail($user, $totalSumDue, $filename, $charge['site_home']);
+		}
 	}
 
 	public function sendNotificationMail($user, $amount, $filename, $senderName) {
@@ -274,7 +292,7 @@ public function sendNotificationMail($user, $amount, $filename, $senderName) {
 
 	private function invoice($user, $reference, $homeGB, $backupGB, $totalAmountDue,
 			$homeAmountDue, $backupAmountDue, $homeSite, $backupSite, $groupUsagesGB,
-			$groupCharges, $podsUse){
+			$groupCharges, $podsUse, $dryrun=false){
 
 		\OCP\Util::writeLog('Files_Accounting', 'Billing user: '.$user.' for '.$this->billingMonthName, \OCP\Util::WARN);
 
@@ -294,9 +312,8 @@ private function invoice($user, $reference, $homeGB, $backupGB, $totalAmountDue,
 			);
 		}
 		foreach($podsUse['charges'] as $image=>$cost){
-			array_push($articles, array('item'=>'Pod image '.$image. '  '.$podsUse['seconds'][$image].
-					' seconds, '.$this->billingMonthName.' - '.$this->billingYear,
-					'price'=>$cost)
+			array_push($articles, array('item'=>preg_replace('|^sciencedata/|', '', $image). '  '.self::secondsToTime($podsUse['seconds'][$image]),
+					'price'=>round($cost, 2))
 					);
 		}
 		\OCP\Util::writeLog('Files_Accounting', 'HOME: '.$homeGB.' '.$homeAmountDue.' '.$homeSite, \OCP\Util::WARN);
@@ -306,7 +323,7 @@ private function invoice($user, $reference, $homeGB, $backupGB, $totalAmountDue,
 		for ($i=0; $i<count($articles); $i++){
 			$total += $articles[$i]['price'];
 		}*/
-		$filename = $reference.'.pdf';
+		$filename = ($dryrun?'test-':'').$reference.'.pdf';
 		$userEmail = \OCP\Config::getUserValue($user, 'settings', 'email');
 		$userRealName = User::getDisplayName($user);
 		$fromEmail = \OCA\Files_Accounting\Storage_Lib::getIssuerEmail();
@@ -387,5 +404,44 @@ private function writeInvoice($user, $userEmail, $userRealName, $email, $address
 		}
 		$pdf->Output($path.'/'.$filename, 'F');
 	}
+
+	// https://stackoverflow.com/questions/8273804/convert-seconds-into-days-hours-minutes-and-seconds
+	private static function secondsToTime($inputSeconds) {
+		$secondsInAMinute = 60;
+		$secondsInAnHour = 60 * $secondsInAMinute;
+		$secondsInADay = 24 * $secondsInAnHour;
+
+		// Extract days
+		$days = floor($inputSeconds / $secondsInADay);
+
+		// Extract hours
+		$hourSeconds = $inputSeconds % $secondsInADay;
+		$hours = floor($hourSeconds / $secondsInAnHour);
+
+		// Extract minutes
+		$minuteSeconds = $hourSeconds % $secondsInAnHour;
+		$minutes = floor($minuteSeconds / $secondsInAMinute);
+
+		// Extract the remaining seconds
+		$remainingSeconds = $minuteSeconds % $secondsInAMinute;
+		$seconds = ceil($remainingSeconds);
+
+		// Format and return
+		$timeParts = [];
+		$sections = [
+				'day' => (int)$days,
+				'hour' => (int)$hours,
+				'min' => (int)$minutes,
+				's' => (int)$seconds,
+		];
+
+		foreach ($sections as $name => $value){
+			if ($value > 0){
+				$timeParts[] = $value. ' '.$name.($value == 1 || $name=='min' || $name=='s' ? '' : 's');
+			}
+		}
+
+		return implode(', ', $timeParts);
+	}
 }
 
diff --git a/lib/storage_lib.php b/lib/storage_lib.php
@@ -91,19 +91,20 @@ public static function getUsageFilePath($user, $year, $group=null){
 	public static function getInvoiceDir($user){
 		return self::getAppDir($user)."/bills";
 	}
-
-	public static function getPodsUsageFilePath($user, $year, $month){
-		$dir = getPodsUsageDir($user);
-		return $dir."/podsusage"."_".$year."_".$month.".txt";
-	}
-
+
 	public static function getPodsUsageDir($user){
 		return self::getAppDir($user)."/pods";
 	}
+
+	public static function getPodsUsageFilePath($user, $year, $month){
+		$dir = self::getPodsUsageDir($user);
+		return $dir."/podsusage"."_".$year."_".$month.".txt";
+	}
 
 	/**
 	 * Read lines in files_accounting/pods/[year]_[month].txt and
-	 * add running_seconds * pod_charge_per_second for each.
+	 * add running_seconds * pod_charge_per_second for each unique pod 
+	 * (last entry with timestamp before cycle_day).
 	 * @param unknown $user
 	 */
 	public static function getPodsMonthlyUse($user, $year=null, $month=null){
@@ -120,9 +121,17 @@ public static function getPodsMonthlyUse($user, $year=null, $month=null){
 			return false;
 		}
 		$lines = file($usageFilePath);
-		foreach ($lines as $line) {
-			$row = explode(" ", $line);
-			if(!empty($row) && $row[0] == $user){
+		// Iterate backwards, so we pick the last entry for a given pod and ignore previous
+		$accounted_pods = [];
+		$index = count($lines);
+		while($index) {
+			$line = $lines[--$index];
+			$row = explode("|", $line);
+			$podName = $row[4];
+			$cycle_day = $row[9];
+			$billingDateSeconds = mktime(0, 0, 0, $cycle_day, $month, $year);
+			$report_timestamp = $row[10];
+			if(!empty($row) && $row[0]==$user && empty($accounted_pods[$podName]) && $report_timestamp<$billingDateSeconds /*only count usage before, but as close as possible to 0:00 on billing day*/){
 				$imageName = $row[3];
 				$runningSeconds = $row[8];
 				if(!empty($imageName) && !empty($runningSeconds)){
@@ -131,13 +140,17 @@ public static function getPodsMonthlyUse($user, $year=null, $month=null){
 						$ret['charges'][$imageName] = 0.0;
 					}
 					$ret['seconds'][$imageName] += $runningSeconds;
+					$accounted_pods[$podName] = 1;
 					foreach($chargePatterns as $pattern => $price){
-						if(preg_match($pattern, $imageName)){
+						if(preg_match('|'.$pattern.'|', $imageName)){
+							// we've exceeded the free tier, charge all $runningSeconds
 							if($totalSeconds>$freeSeconds){
 								$charge = ((float)$price) * $runningSeconds;
 								$ret['charges'][$imageName] += $charge;
 								$totalCharge += $charge;
 							}
+							// We've not yet exceeded the free tier, but will with these $runningSeconds,
+							// charge the exceeded $runningSeconds
 							elseif($totalSeconds+$runningSeconds>$freeSeconds){
 								$charge = ((float)$price) * ($runningSeconds-($freeSeconds-$totalSeconds));
 								$ret['charges'][$imageName] += $charge;

diff --git a/report_pod_usage.php b/report_pod_usage.php
@@ -13,6 +13,7 @@
 $node_name = $_REQUEST['node_name'];
 $node_ip = $_REQUEST['node_ip'];
 $pod_name = $_REQUEST['pod_name'];
+$image_name = $_REQUEST['image_name'];
 $pod_ip = $_REQUEST['pod_ip'];
 $start_time = $_REQUEST['start_time'];
 $end_time = $_REQUEST['end_time'];
@@ -26,25 +27,29 @@
 $timestamp = time();
 
 // Write file files_accounting/pods/year_month_pod_id.txt
-#USER NODE_NAME NODE_IP POD_NAME POD_IP START_TIME END_TIME RUNNING_SECONDS TIMESTAMP CYCLE_DAY
+#USER NODE_NAME NODE_IP IMAGE_NAME POD_NAME POD_IP START_TIME END_TIME RUNNING_SECONDS TIMESTAMP CYCLE_DAY
 
 
 $dirPath = \OCA\Files_Accounting\Storage_Lib::getPodsUsageDir($user);
 if(!file_exists($dirPath)){
 	mkdir($dirPath, 0777, false);
 }
-$filePath = $dirPath . "podsusage_" . $year . "_" . $month .".txt";
+$filePath = $dirPath . "/podsusage_" . $year . "_" . $month .".txt";
 
-$data = "$user $node_name $node_ip $pod_name $pod_ip $start_time $end_time $running_seconds $timestamp $cycle_day";
+$data = $user."|".$node_name."|".$node_ip."|".$image_name."|".$pod_name."|".$pod_ip."|".$start_time."|".$end_time."|".$running_seconds."|".$cycle_day."|".$timestamp."|".$day."\n";
+
+# Notice that when reporting to this endpoint several times in a period,
+# the same pod can/will occur several times in $filePath.
+# 
 
 for($i=0; $i<3; ++$i){
-	$ret = file_put_contents($filePath, $data, FILE_APPEND | LOCK_EX);
+	$ret = file_put_contents($filePath, $data, FILE_APPEND);
 	if($ret){
 		break;
 	}
 	sleep(1);
 }
 
-OCP\JSON::encodedPrint($ret);
+OCP\JSON::encodedPrint(['message'=>'Wrote file '.$filePath, 'status'=>$ret]);