diff --git a/packages/pulp-geocsv/src/FromCsvHandler.php b/packages/pulp-geocsv/src/FromCsvHandler.php index 934b358..6951206 100644 --- a/packages/pulp-geocsv/src/FromCsvHandler.php +++ b/packages/pulp-geocsv/src/FromCsvHandler.php @@ -22,12 +22,13 @@ public function onFile(File $file): void $qc = $this->cp->quoteChar; $rows = explode($ls, $csv); + $rows = array_values(array_filter($rows, static fn(string $row): bool => $row !== '')); $rows = array_map(static function ($row) use ($fs, $qc): array { $items = str_getcsv($row, $fs, $qc, ''); - return array_map(stripslashes(...), $items); + return array_map(static fn($item) => stripslashes((string) ($item ?? '')), $items); }, $rows); - $firstRow = array_shift($rows); + $firstRow = array_shift($rows) ?? []; $columnCounter = 0; $columns = array_map(static function ($columnName) use ($columnCounter): string { diff --git a/packages/pulp-geocsv/src/ReprojectHandler.php b/packages/pulp-geocsv/src/ReprojectHandler.php index 8682240..225767c 100644 --- a/packages/pulp-geocsv/src/ReprojectHandler.php +++ b/packages/pulp-geocsv/src/ReprojectHandler.php @@ -120,13 +120,18 @@ protected function readInputGeometry(array $columns, array $row) case ProjectionInfo::TYPE_X_Y: $xIndex = $this->getColumnIndexOrThrow($columns, $input->getColumnNameX()); $yIndex = $this->getColumnIndexOrThrow($columns, $input->getColumnNameY()); - $coords = array_map($parseFloat, [$row[$xIndex], $row[$yIndex]]); + $x = $row[$xIndex] ?? null; + $y = $row[$yIndex] ?? null; + if ($x === null || $x === '' || $y === null || $y === '') { + return null; + } + $coords = array_map($parseFloat, [$x, $y]); return ['type' => 'Point', 'coordinates' => $coords]; case ProjectionInfo::TYPE_XY: $coordinateSeparator = $input->getCoordinateSeparator(); $index = $this->getColumnIndexOrThrow($columns, $input->getColumnNameXY()); - $parts = explode($coordinateSeparator, (string) $row[$index]); + $parts = explode($coordinateSeparator, (string) ($row[$index] ?? '')); [$x, $y] = array_chunk($parts, ceil(count($parts) / 2)); $coords = array_map(static fn($arr): string => implode($coordinateSeparator, $arr), [$x, $y]); $coords = array_map($parseFloat, $coords); @@ -134,7 +139,7 @@ protected function readInputGeometry(array $columns, array $row) case ProjectionInfo::TYPE_WKT: $index = $this->getColumnIndexOrThrow($columns, $input->getColumnNameWKT()); - $wktString = $row[$index]; + $wktString = $row[$index] ?? null; try { $geometry = geoPHP::load($wktString, 'wkt'); if ($geometry !== null) { @@ -149,8 +154,12 @@ protected function readInputGeometry(array $columns, array $row) } } - protected function reproject(array $inputGeometry, array &$errors): ?array + protected function reproject(?array $inputGeometry, array &$errors): ?array { + if ($inputGeometry === null) { + return null; + } + /** @var ProjectionInfo $input */ $input = $this->cp->input; diff --git a/packages/pulp-geocsv/test/CsvHandlerTest.php b/packages/pulp-geocsv/test/CsvHandlerTest.php index a35f4cd..529c6f7 100644 --- a/packages/pulp-geocsv/test/CsvHandlerTest.php +++ b/packages/pulp-geocsv/test/CsvHandlerTest.php @@ -28,6 +28,62 @@ public function testCsvCanBeDecodedAndEncoded(): void $this->assertSame("name;x;y\nBerlin;13.4;52.5\n", $result[0]->content); } + public function testTrailingEmptyLineIsDropped(): void + { + $file = new File('places.csv'); + $file->content = "name;x;y\nBerlin;13.4;52.5\n"; + + $result = Pulp::start() + ->pipe(Pulp::src($file)) + ->pipe(PulpGeoCSV::fromCsv()) + ->run(); + + $this->assertSame(['name', 'x', 'y'], $result[0]->content['columns']); + $this->assertSame([['Berlin', '13.4', '52.5']], $result[0]->content['rows']); + } + + public function testReprojectSkipsRowsMissingCoordinateColumns(): void + { + $file = new File('places.csv'); + $file->content = [ + 'type' => 'text/csv', + 'columns' => ['x', 'y'], + 'rows' => [ + ['13,4', '52,5'], + [''], + ], + ]; + + $input = ProjectionInfo::build( + ProjectionInfo::TYPE_X_Y, + 'x', + 'y', + null, + null, + 'EPSG:4326', + ',', + ',' + ); + $output = ProjectionInfo::build( + ProjectionInfo::TYPE_XY, + null, + null, + 'coords', + null, + 'EPSG:4326', + ',', + '|' + ); + + $result = Pulp::start() + ->pipe(Pulp::src($file)) + ->pipe(PulpGeoCSV::reproject($input, $output)) + ->run(); + + $this->assertSame('13,4|52,5', $result[0]->content['rows'][0][2]); + $this->assertSame('', $result[0]->content['rows'][1][2] ?? ''); + } + public function testReprojectAddsConfiguredOutputColumn(): void { $file = new File('places.csv'); diff --git a/packages/pulp-geojson/src/FromCsvHandler.php b/packages/pulp-geojson/src/FromCsvHandler.php index 50aaef2..b50acee 100644 --- a/packages/pulp-geojson/src/FromCsvHandler.php +++ b/packages/pulp-geojson/src/FromCsvHandler.php @@ -97,23 +97,26 @@ protected function getConstructorParamDefs(): array public function onFile(File $file): void { $csv = $file->content; - $csv = mb_convert_encoding($csv, 'UTF-8', 'ISO-8859-1'); // TODO: make conversion optional + if (is_string($csv) && $csv !== '' && !mb_check_encoding($csv, 'UTF-8')) { + $csv = mb_convert_encoding($csv, 'UTF-8', 'ISO-8859-1'); + } $rows = explode($this->cp->lineSeparator, $csv); + $rows = array_values(array_filter($rows, static fn(string $row): bool => $row !== '')); $rows = array_map(function ($row): array { - $items = str_getcsv($row, $this->cp->fieldSeparator, $this->cp->quoteChar); - return array_map(stripslashes(...), $items); + $items = str_getcsv($row, $this->cp->fieldSeparator, $this->cp->quoteChar, ''); + return array_map(static fn($item) => stripslashes((string) ($item ?? '')), $items); }, $rows); $columnCounter = 0; $columns = array_map(static function ($columnName) use ($columnCounter): string { $columnCounter++; return $columnName === '' || $columnName === '0' ? '_' . $columnCounter : $columnName; - }, array_shift($rows)); + }, array_shift($rows) ?? []); $rowToFeatureMapper = $this->cp->rowToFeatureMapper ?? self::defaultRowToFeatureMapper(...); - $features = array_map(fn($row) => $rowToFeatureMapper($row, $columns), $rows, $columns); + $features = array_map(fn($row) => $rowToFeatureMapper($row, $columns), $rows); // using array_values to fix holes in array, see https://stackoverflow.com/a/2653022 $features = array_values(array_filter($features, static fn($feature): bool => !empty($feature))); diff --git a/packages/pulp-geojson/src/MergeHandler.php b/packages/pulp-geojson/src/MergeHandler.php index f3796a5..3087b45 100644 --- a/packages/pulp-geojson/src/MergeHandler.php +++ b/packages/pulp-geojson/src/MergeHandler.php @@ -68,7 +68,9 @@ public function onEnd(): void 'type' => 'FeatureCollection', 'features' => $this->features, ]; - $data = GeoJsonReproject::setProjection($data, $this->projection); + if ($this->projection !== null) { + $data = GeoJsonReproject::setProjection($data, $this->projection); + } $file = new File($this->cp->fileName); $file->content = $data; diff --git a/packages/pulp-geojson/test/FromCsvHandlerTest.php b/packages/pulp-geojson/test/FromCsvHandlerTest.php new file mode 100644 index 0000000..5cf149f --- /dev/null +++ b/packages/pulp-geojson/test/FromCsvHandlerTest.php @@ -0,0 +1,75 @@ +content = "name;ort\nAlpha;Braunschweig\nBeta;Braunschweig\nGamma;Hannover\n"; + + $res = Pulp::start() + ->pipe(Pulp::src($file)) + ->pipe(PulpGeoJSON::fromCsv()) + ->run(); + + $names = array_map( + static fn(array $feature): string => (string) ($feature['properties']['name'] ?? ''), + $res[0]->content['features'] + ); + $this->assertSame(['Alpha', 'Beta', 'Gamma'], $names); + } + + public function testKeepsUtf8Umlauts(): void + { + $file = new File('streets.csv'); + $file->content = "name;ort\nOkerinsel;Straße\n"; + + $res = Pulp::start() + ->pipe(Pulp::src($file)) + ->pipe(PulpGeoJSON::fromCsv()) + ->run(); + + $this->assertSame( + 'Straße', + $res[0]->content['features'][0]['properties']['ort'] + ); + } + + public function testMapsRowWhenThereAreMoreColumnsThanRows(): void + { + $file = new File('one.csv'); + $file->content = "name;a;b;c;d\nonly;;;;\n"; + + $res = Pulp::start() + ->pipe(Pulp::src($file)) + ->pipe(PulpGeoJSON::fromCsv()) + ->run(); + + $this->assertCount(1, $res[0]->content['features']); + $this->assertSame('only', $res[0]->content['features'][0]['properties']['name']); + } + + public function testTrailingEmptyLineDoesNotFail(): void + { + $file = new File('stations.csv'); + $file->content = "name;ort\nAlpha;Braunschweig\n"; + + $res = Pulp::start() + ->pipe(Pulp::src($file)) + ->pipe(PulpGeoJSON::fromCsv()) + ->pipe(PulpGeoJSON::merge('stations.geojson')) + ->run(); + + $this->assertCount(1, $res[0]->content['features']); + $this->assertSame('Alpha', $res[0]->content['features'][0]['properties']['name']); + } +} diff --git a/packages/pulp-geojson/test/MergeHandlerTest.php b/packages/pulp-geojson/test/MergeHandlerTest.php index bacb102..5d50b2e 100644 --- a/packages/pulp-geojson/test/MergeHandlerTest.php +++ b/packages/pulp-geojson/test/MergeHandlerTest.php @@ -5,6 +5,7 @@ namespace OpenMapsight\pulpgeojson\dev\test; use OpenMapsight\Pulp; +use OpenMapsight\pulp\File; use OpenMapsight\PulpGeoJSON; use OpenMapsight\PulpJSON; use PHPUnit\Framework\TestCase; @@ -29,4 +30,29 @@ public function test(): void $this->assertCount(1, $res); TestUtils::assertJsonSameFile('expected.merge.geojson', $res[0]->content); } + + public function testMergeWithoutCrsDoesNotTypeError(): void + { + $file = new File('points.geojson'); + $file->content = [ + 'type' => 'FeatureCollection', + 'features' => [ + [ + 'type' => 'Feature', + 'geometry' => ['type' => 'Point', 'coordinates' => [10.5, 52.2]], + 'properties' => ['name' => 'A'], + ], + ], + ]; + + $res = Pulp::start() + ->pipe(Pulp::src($file)) + ->pipe(PulpGeoJSON::merge('output.geojson')) + ->run(); + + $this->assertCount(1, $res); + $this->assertSame('FeatureCollection', $res[0]->content['type']); + $this->assertCount(1, $res[0]->content['features']); + $this->assertSame('A', $res[0]->content['features'][0]['properties']['name']); + } }