Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 3 additions & 2 deletions packages/pulp-geocsv/src/FromCsvHandler.php
Original file line number Diff line number Diff line change
Expand Up @@ -22,12 +22,13 @@ public function onFile(File $file): void
$qc = $this->cp->quoteChar;

$rows = explode($ls, $csv);
$rows = array_values(array_filter($rows, static fn(string $row): bool => $row !== ''));
$rows = array_map(static function ($row) use ($fs, $qc): array {
$items = str_getcsv($row, $fs, $qc, '');
return array_map(stripslashes(...), $items);
return array_map(static fn($item) => stripslashes((string) ($item ?? '')), $items);
}, $rows);

$firstRow = array_shift($rows);
$firstRow = array_shift($rows) ?? [];

$columnCounter = 0;
$columns = array_map(static function ($columnName) use ($columnCounter): string {
Expand Down
17 changes: 13 additions & 4 deletions packages/pulp-geocsv/src/ReprojectHandler.php
Original file line number Diff line number Diff line change
Expand Up @@ -120,21 +120,26 @@ protected function readInputGeometry(array $columns, array $row)
case ProjectionInfo::TYPE_X_Y:
$xIndex = $this->getColumnIndexOrThrow($columns, $input->getColumnNameX());
$yIndex = $this->getColumnIndexOrThrow($columns, $input->getColumnNameY());
$coords = array_map($parseFloat, [$row[$xIndex], $row[$yIndex]]);
$x = $row[$xIndex] ?? null;
$y = $row[$yIndex] ?? null;
if ($x === null || $x === '' || $y === null || $y === '') {
return null;
}
$coords = array_map($parseFloat, [$x, $y]);
return ['type' => 'Point', 'coordinates' => $coords];

case ProjectionInfo::TYPE_XY:
$coordinateSeparator = $input->getCoordinateSeparator();
$index = $this->getColumnIndexOrThrow($columns, $input->getColumnNameXY());
$parts = explode($coordinateSeparator, (string) $row[$index]);
$parts = explode($coordinateSeparator, (string) ($row[$index] ?? ''));
[$x, $y] = array_chunk($parts, ceil(count($parts) / 2));
$coords = array_map(static fn($arr): string => implode($coordinateSeparator, $arr), [$x, $y]);
$coords = array_map($parseFloat, $coords);
return ['type' => 'Point', 'coordinates' => $coords];

case ProjectionInfo::TYPE_WKT:
$index = $this->getColumnIndexOrThrow($columns, $input->getColumnNameWKT());
$wktString = $row[$index];
$wktString = $row[$index] ?? null;
try {
$geometry = geoPHP::load($wktString, 'wkt');
if ($geometry !== null) {
Expand All @@ -149,8 +154,12 @@ protected function readInputGeometry(array $columns, array $row)
}
}

protected function reproject(array $inputGeometry, array &$errors): ?array
protected function reproject(?array $inputGeometry, array &$errors): ?array
{
if ($inputGeometry === null) {
return null;
}

/** @var ProjectionInfo $input */
$input = $this->cp->input;

Expand Down
56 changes: 56 additions & 0 deletions packages/pulp-geocsv/test/CsvHandlerTest.php
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,62 @@ public function testCsvCanBeDecodedAndEncoded(): void
$this->assertSame("name;x;y\nBerlin;13.4;52.5\n", $result[0]->content);
}

public function testTrailingEmptyLineIsDropped(): void
{
$file = new File('places.csv');
$file->content = "name;x;y\nBerlin;13.4;52.5\n";

$result = Pulp::start()
->pipe(Pulp::src($file))
->pipe(PulpGeoCSV::fromCsv())
->run();

$this->assertSame(['name', 'x', 'y'], $result[0]->content['columns']);
$this->assertSame([['Berlin', '13.4', '52.5']], $result[0]->content['rows']);
}

public function testReprojectSkipsRowsMissingCoordinateColumns(): void
{
$file = new File('places.csv');
$file->content = [
'type' => 'text/csv',
'columns' => ['x', 'y'],
'rows' => [
['13,4', '52,5'],
[''],
],
];

$input = ProjectionInfo::build(
ProjectionInfo::TYPE_X_Y,
'x',
'y',
null,
null,
'EPSG:4326',
',',
','
);
$output = ProjectionInfo::build(
ProjectionInfo::TYPE_XY,
null,
null,
'coords',
null,
'EPSG:4326',
',',
'|'
);

$result = Pulp::start()
->pipe(Pulp::src($file))
->pipe(PulpGeoCSV::reproject($input, $output))
->run();

$this->assertSame('13,4|52,5', $result[0]->content['rows'][0][2]);
$this->assertSame('', $result[0]->content['rows'][1][2] ?? '');
}

public function testReprojectAddsConfiguredOutputColumn(): void
{
$file = new File('places.csv');
Expand Down
13 changes: 8 additions & 5 deletions packages/pulp-geojson/src/FromCsvHandler.php
Original file line number Diff line number Diff line change
Expand Up @@ -97,23 +97,26 @@ protected function getConstructorParamDefs(): array
public function onFile(File $file): void
{
$csv = $file->content;
$csv = mb_convert_encoding($csv, 'UTF-8', 'ISO-8859-1'); // TODO: make conversion optional
if (is_string($csv) && $csv !== '' && !mb_check_encoding($csv, 'UTF-8')) {
$csv = mb_convert_encoding($csv, 'UTF-8', 'ISO-8859-1');
}

$rows = explode($this->cp->lineSeparator, $csv);
$rows = array_values(array_filter($rows, static fn(string $row): bool => $row !== ''));
$rows = array_map(function ($row): array {
$items = str_getcsv($row, $this->cp->fieldSeparator, $this->cp->quoteChar);
return array_map(stripslashes(...), $items);
$items = str_getcsv($row, $this->cp->fieldSeparator, $this->cp->quoteChar, '');
return array_map(static fn($item) => stripslashes((string) ($item ?? '')), $items);
}, $rows);

$columnCounter = 0;
$columns = array_map(static function ($columnName) use ($columnCounter): string {
$columnCounter++;
return $columnName === '' || $columnName === '0' ? '_' . $columnCounter : $columnName;
}, array_shift($rows));
}, array_shift($rows) ?? []);

$rowToFeatureMapper = $this->cp->rowToFeatureMapper ?? self::defaultRowToFeatureMapper(...);

$features = array_map(fn($row) => $rowToFeatureMapper($row, $columns), $rows, $columns);
$features = array_map(fn($row) => $rowToFeatureMapper($row, $columns), $rows);

// using array_values to fix holes in array, see https://stackoverflow.com/a/2653022
$features = array_values(array_filter($features, static fn($feature): bool => !empty($feature)));
Expand Down
4 changes: 3 additions & 1 deletion packages/pulp-geojson/src/MergeHandler.php
Original file line number Diff line number Diff line change
Expand Up @@ -68,7 +68,9 @@ public function onEnd(): void
'type' => 'FeatureCollection',
'features' => $this->features,
];
$data = GeoJsonReproject::setProjection($data, $this->projection);
if ($this->projection !== null) {
$data = GeoJsonReproject::setProjection($data, $this->projection);
}

$file = new File($this->cp->fileName);
$file->content = $data;
Expand Down
75 changes: 75 additions & 0 deletions packages/pulp-geojson/test/FromCsvHandlerTest.php
Original file line number Diff line number Diff line change
@@ -0,0 +1,75 @@
<?php

declare(strict_types=1);

namespace OpenMapsight\pulpgeojson\dev\test;

use OpenMapsight\Pulp;
use OpenMapsight\pulp\File;
use OpenMapsight\PulpGeoJSON;
use PHPUnit\Framework\TestCase;

class FromCsvHandlerTest extends TestCase
{
public function testMapsEveryRowWhenThereAreMoreRowsThanColumns(): void
{
$file = new File('stations.csv');
$file->content = "name;ort\nAlpha;Braunschweig\nBeta;Braunschweig\nGamma;Hannover\n";

$res = Pulp::start()
->pipe(Pulp::src($file))
->pipe(PulpGeoJSON::fromCsv())
->run();

$names = array_map(
static fn(array $feature): string => (string) ($feature['properties']['name'] ?? ''),
$res[0]->content['features']
);
$this->assertSame(['Alpha', 'Beta', 'Gamma'], $names);
}

public function testKeepsUtf8Umlauts(): void
{
$file = new File('streets.csv');
$file->content = "name;ort\nOkerinsel;Straße\n";

$res = Pulp::start()
->pipe(Pulp::src($file))
->pipe(PulpGeoJSON::fromCsv())
->run();

$this->assertSame(
'Straße',
$res[0]->content['features'][0]['properties']['ort']
);
}

public function testMapsRowWhenThereAreMoreColumnsThanRows(): void
{
$file = new File('one.csv');
$file->content = "name;a;b;c;d\nonly;;;;\n";

$res = Pulp::start()
->pipe(Pulp::src($file))
->pipe(PulpGeoJSON::fromCsv())
->run();

$this->assertCount(1, $res[0]->content['features']);
$this->assertSame('only', $res[0]->content['features'][0]['properties']['name']);
}

public function testTrailingEmptyLineDoesNotFail(): void
{
$file = new File('stations.csv');
$file->content = "name;ort\nAlpha;Braunschweig\n";

$res = Pulp::start()
->pipe(Pulp::src($file))
->pipe(PulpGeoJSON::fromCsv())
->pipe(PulpGeoJSON::merge('stations.geojson'))
->run();

$this->assertCount(1, $res[0]->content['features']);
$this->assertSame('Alpha', $res[0]->content['features'][0]['properties']['name']);
}
}
26 changes: 26 additions & 0 deletions packages/pulp-geojson/test/MergeHandlerTest.php
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
namespace OpenMapsight\pulpgeojson\dev\test;

use OpenMapsight\Pulp;
use OpenMapsight\pulp\File;
use OpenMapsight\PulpGeoJSON;
use OpenMapsight\PulpJSON;
use PHPUnit\Framework\TestCase;
Expand All @@ -29,4 +30,29 @@ public function test(): void
$this->assertCount(1, $res);
TestUtils::assertJsonSameFile('expected.merge.geojson', $res[0]->content);
}

public function testMergeWithoutCrsDoesNotTypeError(): void
{
$file = new File('points.geojson');
$file->content = [
'type' => 'FeatureCollection',
'features' => [
[
'type' => 'Feature',
'geometry' => ['type' => 'Point', 'coordinates' => [10.5, 52.2]],
'properties' => ['name' => 'A'],
],
],
];

$res = Pulp::start()
->pipe(Pulp::src($file))
->pipe(PulpGeoJSON::merge('output.geojson'))
->run();

$this->assertCount(1, $res);
$this->assertSame('FeatureCollection', $res[0]->content['type']);
$this->assertCount(1, $res[0]->content['features']);
$this->assertSame('A', $res[0]->content['features'][0]['properties']['name']);
}
}
Loading