Skip to content

Commit

Permalink
✨ fetch station labels from wikidata (#2828)
Browse files Browse the repository at this point in the history
  • Loading branch information
@MrKrisKrisu
MrKrisKrisu authored Aug 7, 2024
1 parent e9be316 commit 17aeffa
Show file tree
Hide file tree
Showing 27 changed files with 495 additions and 252 deletions.
2 changes: 2 additions & 0 deletions .gitignore
View file
Original file line number Diff line number Diff line change
Expand Up @@ -144,3 +144,5 @@ result*

# laravel-vue-i18n
lang/php_*.json

*.sqlite
1 change: 1 addition & 0 deletions .idea/php.xml
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

3 changes: 2 additions & 1 deletion .idea/trwl.iml
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

19 changes: 0 additions & 19 deletions app/Console/Commands/WikidataFetcher.php
View file

This file was deleted.

1 change: 0 additions & 1 deletion app/Console/Kernel.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,6 @@ class Kernel extends ConsoleKernel
protected function schedule(Schedule $schedule): void {
//every minute
$schedule->command(RefreshCurrentTrips::class)->withoutOverlapping()->everyMinute();
$schedule->command(WikidataFetcher::class)->withoutOverlapping()->everyMinute();

//every five minutes
$schedule->command(CacheLeaderboard::class)->withoutOverlapping()->everyFiveMinutes();
Expand Down
53 changes: 53 additions & 0 deletions app/Dto/Wikidata/WikidataObject.php
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,53 @@
<?php declare(strict_types=1);

namespace App\Dto\Wikidata;

use Illuminate\Database\Eloquent\ModelNotFoundException;
use JsonException;

class WikidataObject
{

public readonly string $qId;
private $rawData;

/**
* @param string $qId
*
* @return self
* @throws JsonException|ModelNotFoundException
*/
public static function fetch(string $qId): self {
if (!str_starts_with($qId, 'Q')) {
throw new \InvalidArgumentException('Invalid QID');
}
$instance = new self();
$instance->qId = $qId;
$json = json_decode(file_get_contents('https://www.wikidata.org/wiki/Special:EntityData/' . $qId . '.json'), true, 512, JSON_THROW_ON_ERROR);

if (!isset($json['entities'][$qId])) {
throw new ModelNotFoundException('Entity not found');
}

$instance->rawData = $json['entities'][$qId];

return $instance;
}

public function getLabels(): array {
return $this->rawData['labels'] ?? [];
}

public function getLabel(string $language = 'en'): ?string {
return $this->rawData['labels'][$language]['value'] ?? null;
}

public function getDescription(string $language = 'en'): ?string {
return $this->rawData['descriptions'][$language]['value'] ?? null;
}

public function getClaims(string $property): array {
return $this->rawData['claims'][$property] ?? [];
}

}
8 changes: 0 additions & 8 deletions app/Http/Controllers/API/v1/StationController.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,15 +2,13 @@

namespace App\Http\Controllers\API\v1;

use App\Http\Controllers\Backend\Wikidata\WikidataFetchController;
use App\Http\Resources\StationResource;
use App\Models\Checkin;
use App\Models\Event;
use App\Models\EventSuggestion;
use App\Models\Station;
use App\Models\Stopover;
use App\Models\Trip;
use App\Models\WikidataEntity;
use Illuminate\Auth\Access\AuthorizationException;
use Illuminate\Http\JsonResponse;
use Illuminate\Http\Request;
Expand All @@ -37,18 +35,12 @@ public function update(Request $request, int $id): StationResource {

$validated = $request->validate([
'ibnr' => ['nullable', 'numeric', 'unique:train_stations,ibnr,' . $station->id],
'wikidata_id' => ['nullable', 'string', 'max:255'],
'rilIdentifier' => ['nullable', 'string', 'max:10'],
'name' => ['nullable', 'string', 'max:255'],
'latitude' => ['nullable', 'numeric', 'between:-90,90'],
'longitude' => ['nullable', 'numeric', 'between:-180,180'],
]);

if (isset($validated['wikidata_id'])) {
$wikidataEntity = WikidataEntity::updateOrCreate(['id' => $validated['wikidata_id']]);
WikidataFetchController::fetchEntity($wikidataEntity->fresh());
}

$station->update($validated);
return new StationResource($station);
}
Expand Down
12 changes: 6 additions & 6 deletions app/Http/Controllers/Backend/User/DashboardController.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,9 +24,9 @@ public static function getPrivateDashboard(User $user): Paginator {
'checkin',
'tags',
'mentions.mentioned',
'checkin.originStopover.station',
'checkin.destinationStopover.station',
'checkin.trip.stopovers.station'
'checkin.originStopover.station.names',
'checkin.destinationStopover.station.names',
'checkin.trip.stopovers.station.names'
])
->join('train_checkins', 'train_checkins.status_id', '=', 'statuses.id')
->select('statuses.*')
Expand All @@ -53,9 +53,9 @@ public static function getGlobalDashboard(User $user): Paginator {
'checkin',
'mentions.mentioned',
'tags',
'checkin.originStopover.station',
'checkin.destinationStopover.station',
'checkin.trip.stopovers.station'
'checkin.originStopover.station.names',
'checkin.destinationStopover.station.names',
'checkin.trip.stopovers.station.names'
])
->join('train_checkins', 'train_checkins.status_id', '=', 'statuses.id')
->join('users', 'statuses.user_id', '=', 'users.id')
Expand Down
40 changes: 0 additions & 40 deletions app/Http/Controllers/Backend/Wikidata/WikidataFetchController.php
View file

This file was deleted.

62 changes: 62 additions & 0 deletions app/Http/Controllers/Frontend/Admin/StationController.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,8 +5,11 @@
use App\Dto\Coordinate;
use App\Http\Controllers\Controller;
use App\Models\Station;
use App\Models\StationName;
use App\Objects\LineSegment;
use App\Services\Wikidata\WikidataQueryService;
use Illuminate\Http\Request;
use Illuminate\Support\Facades\Log;
use Illuminate\View\View;

class StationController extends Controller
Expand Down Expand Up @@ -51,4 +54,63 @@ public function renderStation(int $id): View {
'stationsWithSameIfopt' => $stationsWithSameIfopt ?? [],
]);
}

/**
* !!!! Experimental Backend Function !!!!
* Fetches the Wikidata information for a station.
* Try to find matching Wikidata entity for the station and fetch it.
* Needs to be cleaned up and refactored, if it should be used consistently.
* Little testing if it works as expected.
*/
public function fetchWikidata(int $id): void {
$station = Station::findOrFail($id);
$this->authorize('update', $station);

// P054 = IBNR
$sparqlQuery = <<<SPARQL
SELECT ?item WHERE { ?item wdt:P954 "{$station->ibnr}". }
SPARQL;

$objects = (new WikidataQueryService())->setQuery($sparqlQuery)->execute()->getObjects();
if (count($objects) > 1) {
Log::debug('More than one object found for station ' . $station->ibnr . ' (' . $station->id . ') - skipping');
return;
}

if (empty($objects)) {
Log::debug('No object found for station ' . $station->ibnr . ' (' . $station->id . ') - skipping');
return;
}

$object = $objects[0];
$station->update(['wikidata_id' => $object->qId]);
Log::debug('Fetched object ' . $object->qId . ' for station ' . $station->name . ' (Trwl-ID: ' . $station->id . ')');

$ifopt = $object->getClaims('P12393')[0]['mainsnak']['datavalue']['value'] ?? null;
if ($station->ifopt_a === null && $ifopt !== null) {
$splitIfopt = explode(':', $ifopt);
$station->update([
'ifopt_a' => $splitIfopt[0] ?? null,
'ifopt_b' => $splitIfopt[1] ?? null,
'ifopt_c' => $splitIfopt[2] ?? null,
]);
}

$rl100 = $object->getClaims('P8671')[0]['mainsnak']['datavalue']['value'] ?? null;
if ($station->rilIdentifier === null && $rl100 !== null) {
$station->update(['rilIdentifier' => $rl100]);
}

foreach ($object->getLabels() as $lang => $label) {
if ($label['value'] === null) {
continue;
}
StationName::updateOrCreate([
'station_id' => $station->id,
'language' => $lang,
], [
'name' => $label['value']
]);
}
}
}
24 changes: 12 additions & 12 deletions app/Http/Controllers/StatusController.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -50,9 +50,9 @@ public static function getStatus(int $statusId): Status {
'user.blockedUsers',
'checkin',
'tags',
'checkin.originStopover.station',
'checkin.destinationStopover.station',
'checkin.trip.stopovers.station',
'checkin.originStopover.station.names',
'checkin.destinationStopover.station.names',
'checkin.trip.stopovers.station.names',
])
->firstOrFail();
}
Expand All @@ -71,9 +71,9 @@ public static function getActiveStatuses(): ?Collection {
'user.blockedByUsers',
'user.blockedUsers',
'user.followers',
'checkin.originStopover.station',
'checkin.destinationStopover.station',
'checkin.trip.stopovers.station',
'checkin.originStopover.station.names',
'checkin.destinationStopover.station.names',
'checkin.trip.stopovers.station.names',
'checkin.trip.polyline',
'tags',
])
Expand Down Expand Up @@ -110,9 +110,9 @@ public static function getLivePositionForStatus(string $ids): array {
'user.blockedByUsers',
'user.blockedUsers',
'user.followers',
'checkin.originStopover.station',
'checkin.destinationStopover.station',
'checkin.trip.stopovers.station',
'checkin.originStopover.station.names',
'checkin.destinationStopover.station.names',
'checkin.trip.stopovers.station.names',
'checkin.trip.polyline',
])
->whereIn('id', $ids)
Expand Down Expand Up @@ -210,8 +210,8 @@ public static function usageByDay(Carbon $date): int {
public static function getStatusesByEvent(Event $event): array {
$statuses = $event->statuses()
->with([
'user.blockedUsers', 'checkin.originStopover.station',
'checkin.destinationStopover.station', 'checkin.trip.stopovers', 'event', 'likes', 'tags',
'user.blockedUsers', 'checkin.originStopover.station.names',
'checkin.destinationStopover.station.names', 'checkin.trip.stopovers', 'event', 'likes', 'tags',
])
->select('statuses.*')
->join('users', 'statuses.user_id', '=', 'users.id')
Expand Down Expand Up @@ -259,7 +259,7 @@ public static function getStatusesByEvent(Event $event): array {
public static function getFutureCheckins(): Paginator {
return auth()->user()->statuses()
->with([
'user', 'checkin.originStopover.station', 'checkin.destinationStopover.station',
'user', 'checkin.originStopover.station.names', 'checkin.destinationStopover.station.names',
'checkin.trip', 'event', 'tags',
])
->orderByDesc('created_at')
Expand Down
2 changes: 1 addition & 1 deletion app/Http/Controllers/TransportController.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -125,7 +125,7 @@ public static function getOverlappingCheckIns(User $user, Carbon $start, Carbon
return collect();
}

$checkInsToCheck = Checkin::with(['Trip.stopovers', 'originStopover.station', 'destinationStopover.station'])
$checkInsToCheck = Checkin::with(['Trip.stopovers', 'originStopover.station.names', 'destinationStopover.station.names'])
->join('statuses', 'statuses.id', '=', 'train_checkins.status_id')
->where('statuses.user_id', $user->id)
->where('departure', '>=', $start->clone()->subDays(3))
Expand Down
6 changes: 3 additions & 3 deletions app/Http/Controllers/UserController.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,9 +45,9 @@ public static function statusesForUser(User $user, int $limit = null): ?Paginato
'user.blockedUsers',
'checkin',
'tags',
'checkin.originStopover.station',
'checkin.destinationStopover.station',
'checkin.trip.stopovers.station',
'checkin.originStopover.station.names',
'checkin.destinationStopover.station.names',
'checkin.trip.stopovers.station.names',
])
->where(function($query) {
$query->whereIn('statuses.visibility', [
Expand Down
Loading

0 comments on commit 17aeffa

Please sign in to comment.