Add Turnstile protection and harden export workflow
This commit is contained in:
+158
-83
@@ -1,116 +1,191 @@
|
||||
<?php
|
||||
include "zefixAPI.php";
|
||||
include "emailSender.php";
|
||||
declare(strict_types=1);
|
||||
|
||||
$maxExecutionTime = 120;
|
||||
$taskDir = 'tasks';
|
||||
$downloadDir = 'download';
|
||||
$minTaskOldness = 10;
|
||||
$latesEndTime = time() + $maxExecutionTime;
|
||||
$smtppassword = getenv("smtppassword");
|
||||
|
||||
if(!is_dir($downloadDir)){
|
||||
mkdir($downloadDir, 0755, true);
|
||||
if (PHP_SAPI !== 'cli') {
|
||||
http_response_code(404);
|
||||
exit;
|
||||
}
|
||||
|
||||
function doRequest($data, $filename, $username, $password)
|
||||
require_once __DIR__ . '/app.php';
|
||||
require_once __DIR__ . '/zefixAPI.php';
|
||||
require_once __DIR__ . '/emailSender.php';
|
||||
|
||||
$maxExecutionTime = app_env_int('TASK_EXECUTOR_MAX_SECONDS', 120, 30, 900);
|
||||
$minimumTaskAge = app_env_int('TASK_MIN_AGE_SECONDS', 10, 0, 300);
|
||||
$latestEndTime = time() + $maxExecutionTime;
|
||||
$smtpPassword = app_env('smtppassword');
|
||||
|
||||
app_migrate_legacy_tasks();
|
||||
$downloadRetention = app_env_int('DOWNLOAD_RETENTION_HOURS', 48, 1, 720) * 3600;
|
||||
$staleTaskRetention = app_env_int('STALE_TASK_RETENTION_HOURS', 168, 24, 2160) * 3600;
|
||||
app_cleanup_directory(app_download_dir(), $downloadRetention, 'csv');
|
||||
app_cleanup_directory(app_task_dir(), $staleTaskRetention, 'json');
|
||||
app_cleanup_directory(__DIR__ . DIRECTORY_SEPARATOR . 'download', $downloadRetention, 'csv');
|
||||
|
||||
$executorLock = fopen(app_private_dir() . DIRECTORY_SEPARATOR . 'executor.lock', 'c+');
|
||||
if ($executorLock === false || !flock($executorLock, LOCK_EX | LOCK_NB)) {
|
||||
fwrite(STDOUT, "Another executor is already running.\n");
|
||||
exit(0);
|
||||
}
|
||||
|
||||
/**
|
||||
* @return int|false|null Next offset, false when complete, null on a temporary error.
|
||||
*/
|
||||
function doRequest(array $data, string $filename, string $username, string $password): int|false|null
|
||||
{
|
||||
$response = sendAPICompanySearchRequest($username, $password, $data);
|
||||
$responseObject = json_decode($response, true);
|
||||
|
||||
if(array_key_exists("error", $responseObject)) {
|
||||
return false;
|
||||
if (!is_string($response) || $response === '') {
|
||||
return null;
|
||||
}
|
||||
|
||||
$companyArray = $responseObject['list'];
|
||||
$companyData = array();
|
||||
$responseObject = json_decode($response, true);
|
||||
if (!is_array($responseObject) || array_key_exists('error', $responseObject) || !is_array($responseObject['list'] ?? null)) {
|
||||
error_log('ZEFIX search returned an invalid or error response.');
|
||||
return null;
|
||||
}
|
||||
|
||||
$companyData = [];
|
||||
foreach ($responseObject['list'] as $company) {
|
||||
if (!is_array($company) || !isset($company['uid'])) {
|
||||
continue;
|
||||
}
|
||||
|
||||
$detailsResponse = sendAPICompanyInfoRequest($username, $password, (string)$company['uid']);
|
||||
$details = is_string($detailsResponse) ? json_decode($detailsResponse, true) : null;
|
||||
$companyFullData = is_array($details) && is_array($details[0] ?? null) ? $details[0] : null;
|
||||
if ($companyFullData === null) {
|
||||
continue;
|
||||
}
|
||||
|
||||
foreach ($companyArray as $company) {
|
||||
$companyFullData = json_decode(sendAPICompanyInfoRequest($username, $password, $company['uid']), true)[0];
|
||||
$companyData[] = [
|
||||
$companyFullData['name'],
|
||||
$companyFullData['address']['careOf'],
|
||||
$companyFullData['address']['street'],
|
||||
$companyFullData['address']['houseNumber'],
|
||||
$companyFullData['address']['swissZipCode'],
|
||||
$companyFullData['address']['city'],
|
||||
$companyFullData['uid'],
|
||||
$companyFullData['legalSeat'],
|
||||
$companyFullData['legalForm']['name']['de'],
|
||||
$companyFullData['status'],
|
||||
$companyFullData['sogcDate'],
|
||||
$companyFullData['deletionDate']
|
||||
$companyFullData['name'] ?? '',
|
||||
$companyFullData['address']['careOf'] ?? '',
|
||||
$companyFullData['address']['street'] ?? '',
|
||||
$companyFullData['address']['houseNumber'] ?? '',
|
||||
$companyFullData['address']['swissZipCode'] ?? '',
|
||||
$companyFullData['address']['city'] ?? '',
|
||||
$companyFullData['uid'] ?? '',
|
||||
$companyFullData['legalSeat'] ?? '',
|
||||
$companyFullData['legalForm']['name']['de'] ?? '',
|
||||
$companyFullData['status'] ?? '',
|
||||
$companyFullData['sogcDate'] ?? '',
|
||||
$companyFullData['deletionDate'] ?? '',
|
||||
];
|
||||
}
|
||||
|
||||
if (!file_exists($filename)) {
|
||||
$file = fopen($filename, 'w');
|
||||
if($file) {
|
||||
fputcsv($file, ['name', 'careOf', 'street', 'houseNumber', 'swissZipCode', 'city', 'uid', 'legalSeat', 'legalForm', 'status', 'sogcDate', 'deletionDate'], ',', '"', "\\");
|
||||
}
|
||||
} else {
|
||||
$file = fopen($filename, 'a');
|
||||
$isNewFile = !is_file($filename);
|
||||
$file = fopen($filename, $isNewFile ? 'x' : 'a');
|
||||
if ($file === false) {
|
||||
error_log('Export file could not be opened.');
|
||||
return null;
|
||||
}
|
||||
|
||||
if ($file) {
|
||||
foreach ($companyData as $row){
|
||||
fputcsv($file, $row, ',', '"', "\\");
|
||||
try {
|
||||
if ($isNewFile) {
|
||||
fputcsv($file, ['name', 'careOf', 'street', 'houseNumber', 'swissZipCode', 'city', 'uid', 'legalSeat', 'legalForm', 'status', 'sogcDate', 'deletionDate'], ',', '"', '\\');
|
||||
@chmod($filename, 0600);
|
||||
}
|
||||
foreach ($companyData as $row) {
|
||||
fputcsv($file, $row, ',', '"', '\\');
|
||||
}
|
||||
} finally {
|
||||
fclose($file);
|
||||
}
|
||||
|
||||
if($responseObject['hasMoreResults']) {
|
||||
return $responseObject['maxOffset'];
|
||||
} else {
|
||||
return ($responseObject['hasMoreResults'] ?? false)
|
||||
? (int)($responseObject['maxOffset'] ?? 0)
|
||||
: false;
|
||||
}
|
||||
|
||||
function saveTask(string $filename, array $task): bool
|
||||
{
|
||||
$temporary = $filename . '.tmp';
|
||||
$written = file_put_contents($temporary, json_encode($task, JSON_THROW_ON_ERROR), LOCK_EX);
|
||||
if ($written === false || !rename($temporary, $filename)) {
|
||||
@unlink($temporary);
|
||||
return false;
|
||||
}
|
||||
@chmod($filename, 0600);
|
||||
return true;
|
||||
}
|
||||
|
||||
while($latesEndTime - time() > 30){
|
||||
if(is_dir($taskDir)) {
|
||||
$taskfiles = scandir($taskDir);
|
||||
$taskfiles = array_diff($taskfiles, array('.', '..'));
|
||||
sort($taskfiles);
|
||||
try {
|
||||
while ($latestEndTime - time() > 30) {
|
||||
$taskFiles = glob(app_task_dir() . DIRECTORY_SEPARATOR . '*.json') ?: [];
|
||||
sort($taskFiles, SORT_STRING);
|
||||
|
||||
if(count($taskfiles) > 0 && intval(explode("-", $taskfiles[0])[0]) + $minTaskOldness < time()) {
|
||||
$taskString = file_get_contents($taskDir.'/'.$taskfiles[0]);
|
||||
$task = json_decode($taskString, true);
|
||||
if ($taskFiles === []) {
|
||||
fwrite(STDOUT, "Nothing to do.\n");
|
||||
break;
|
||||
}
|
||||
|
||||
if(count($task['requests']) > 0) {
|
||||
$csvFile = str_replace(".json", ".csv", $downloadDir.'/'.$taskfiles[0]);
|
||||
$taskFilename = $taskFiles[0];
|
||||
$modified = filemtime($taskFilename);
|
||||
if ($modified !== false && $modified + $minimumTaskAge > time()) {
|
||||
sleep(min(5, max(1, ($modified + $minimumTaskAge) - time())));
|
||||
continue;
|
||||
}
|
||||
|
||||
$nextOffset = doRequest($task['requests'][0], $csvFile, $username, $password);
|
||||
$task = json_decode((string)file_get_contents($taskFilename), true);
|
||||
if (!is_array($task) || !is_array($task['requests'] ?? null) || !filter_var($task['email'] ?? '', FILTER_VALIDATE_EMAIL)) {
|
||||
error_log('Invalid task quarantined: ' . basename($taskFilename));
|
||||
@rename($taskFilename, $taskFilename . '.invalid');
|
||||
continue;
|
||||
}
|
||||
|
||||
if($nextOffset){
|
||||
$task['requests'][0]['offset'] = $nextOffset;
|
||||
} else {
|
||||
array_shift($task['requests']);
|
||||
}
|
||||
if (!preg_match('/^[a-f0-9]{32}$/', (string)($task['downloadToken'] ?? ''))) {
|
||||
$task['downloadToken'] = bin2hex(random_bytes(16));
|
||||
|
||||
$taskString = json_encode($task);
|
||||
$taskfile = fopen($taskDir.'/'.$taskfiles[0], 'w');
|
||||
|
||||
if($taskfile){
|
||||
fwrite($taskfile, $taskString);
|
||||
fclose($taskfile);
|
||||
}
|
||||
} else {
|
||||
$csvFile = str_replace(".json", ".csv", $downloadDir.'/'.$taskfiles[0]);
|
||||
|
||||
unlink($taskDir.'/'.$taskfiles[0]);
|
||||
|
||||
echo "Task File deleted";
|
||||
echo "<br>";
|
||||
|
||||
sendEmail($task['email'], $csvFile, $smtppassword);
|
||||
// Preserve partially generated exports from the previous public-download layout.
|
||||
$legacyCsv = __DIR__ . DIRECTORY_SEPARATOR . 'download' . DIRECTORY_SEPARATOR . pathinfo($taskFilename, PATHINFO_FILENAME) . '.csv';
|
||||
if (is_file($legacyCsv)) {
|
||||
@rename($legacyCsv, app_download_path($task['downloadToken']));
|
||||
}
|
||||
if (!saveTask($taskFilename, $task)) {
|
||||
error_log('Could not migrate legacy task metadata.');
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
$csvFile = app_download_path((string)$task['downloadToken']);
|
||||
if ($task['requests'] !== []) {
|
||||
$nextOffset = doRequest($task['requests'][0], $csvFile, (string)$username, (string)$password);
|
||||
if ($nextOffset === null) {
|
||||
fwrite(STDOUT, "Temporary API error; task retained for retry.\n");
|
||||
break;
|
||||
}
|
||||
|
||||
if ($nextOffset !== false) {
|
||||
$task['requests'][0]['offset'] = $nextOffset;
|
||||
} else {
|
||||
array_shift($task['requests']);
|
||||
}
|
||||
|
||||
if (!saveTask($taskFilename, $task)) {
|
||||
error_log('Could not save task progress.');
|
||||
break;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!is_file($csvFile)) {
|
||||
$emptyFile = fopen($csvFile, 'x');
|
||||
if ($emptyFile !== false) {
|
||||
fputcsv($emptyFile, ['name', 'careOf', 'street', 'houseNumber', 'swissZipCode', 'city', 'uid', 'legalSeat', 'legalForm', 'status', 'sogcDate', 'deletionDate'], ',', '"', '\\');
|
||||
fclose($emptyFile);
|
||||
@chmod($csvFile, 0600);
|
||||
}
|
||||
}
|
||||
|
||||
if (sendEmail((string)$task['email'], (string)$task['downloadToken'], $smtpPassword)) {
|
||||
unlink($taskFilename);
|
||||
fwrite(STDOUT, 'Completed ' . basename($taskFilename) . ".\n");
|
||||
} else {
|
||||
echo "nothing to do after: ".strval(time() - ($latesEndTime - $maxExecutionTime));
|
||||
echo "sleeping 10 seconds";
|
||||
echo "<br>";
|
||||
sleep(10);
|
||||
fwrite(STDOUT, "Email delivery failed; task retained for retry.\n");
|
||||
break;
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
flock($executorLock, LOCK_UN);
|
||||
fclose($executorLock);
|
||||
}
|
||||
|
||||
?>
|
||||
|
||||
Reference in New Issue
Block a user