Geoid .grd file parser

Error: '.htmlspecialchars($error).'

'; fclose($filehandle); exit; } $batchdata = array(); } } if( !( $databasecount % 1000 ) ) { print ''; ob_flush(); flush(); } } $dbhandle = null; $databasecount = 0; $maxsizereadable = ini_get('upload_max_filesize'); $batchdata = array(); if( $override_file_name || ( isset($_FILES['grdfile']) && $_FILES['grdfile']['error'] == UPLOAD_ERR_OK && $_FILES['grdfile']['tmp_name'] ) ) { //work out how much memory to allocate - these values are based on a 9.17 MB file, EGM96 ww15mgh.grd if( $_POST['dataaction'] == 'render' || $_POST['dataaction'] == 'renderaddlowmem' ) { //uses 11 times the size of the file ini_set('memory_limit','128M'); } elseif( $_POST['dataaction'] == 'renderadd' || $_POST['dataaction'] == 'add' ) { //uses 47 times the size of the file ini_set('memory_limit','512M'); } if( $_POST['dataaction'] == 'renderadd' || $_POST['dataaction'] == 'renderaddlowmem' || $_POST['dataaction'] == 'add' || $_POST['dataaction'] == 'addlowmem' ) { //it takes a lot of time to add a lot of database entries - adjust it here set_time_limit( 60 * 60 ); if( $_POST['dataaction'] == 'renderaddlowmem' || $_POST['dataaction'] == 'addlowmem' ) { /* THIS IS WHERE YOU CONNECT TO THE DATABASE (CONNECT FURTHER DOWN IF USING renderadd OR add) $dbhandle = your database connection handle Set primary keys according to the query method you will use, indexes or lat-long. */ $dbhandle = @database_connect_command( 'somehost', 'someuser', 'somepassword', 'somedatabase' ); if( !$dbhandle ) { print '

ERROR: could not open database

'; } database_query_command( 'CREATE TABLE `'.addslashes($_POST['grdname']).'` ( recordindex int(10), latindex INT(5), longindex INT(5), latitude FLOAT, longitude FLOAT, shift FLOAT, PRIMARY KEY( recordindex ) ) CHARSET=utf8', $dbhandle ); if( $error = database_error_command() ) { print '

Error: '.htmlspecialchars($error).'

'; fclose($filehandle); exit; } } } else { set_time_limit(60); } print '

Processing. Please wait...

'; ob_flush(); flush(); $filename = $override_file_name ? $override_file_name : $_FILES['grdfile']['tmp_name']; $starttime = microtime(true); $filehandle = @fopen( $filename, 'r' ); $haderror = false; $alldata = array(); $headerread = false; $linenumber = 0; $datacount = 0; $highest = -INF; $lowest = INF; $filebytes = 0; $latmin = $latmax = $longmin = $longmax = $latspacing = $longspacing = $latstepdp = $longstepdp = $latcount = $longcount = null; if( $filehandle ) { $numreg = "(-?\d+\.?|-?\d*\.\d+)"; while( ( $line = fgets($filehandle) ) !== false ) { $filebytes += strlen($line); //measured rather than using the upload size, to allow local files to be used instead $linenumber++; //if there is such a thing as comments in these files, they can be removed here if( preg_replace( "/\s+/", '', $line ) ) { if( !$headerread && preg_match( "/^\s*$numreg\s+$numreg\s+$numreg\s+$numreg\s+$numreg\s+$numreg\s*$/", $line, $lineparts ) ) { //read the values out of a header - the first line that is not blank (ignoring comments, if support has been added for comments) $latmin = $lineparts[1]; $latmax = $lineparts[2]; $longmin = $lineparts[3]; $longmax = $lineparts[4]; $latspacingspec = $lineparts[5]; $longspacingspec = $lineparts[6]; $latstepdp = max( strlen( preg_replace( "/^-?\d*\.?/", '', $lineparts[1] * 1 ) ), strlen( preg_replace( "/^-?\d*\.?/", '', $lineparts[5] * 1 ) ) ); $longstepdp = max( strlen( preg_replace( "/^-?\d*\.?/", '', $lineparts[3] * 1 ) ), strlen( preg_replace( "/^-?\d*\.?/", '', $lineparts[6] * 1 ) ) ); $latdiff = $latmax - $latmin; $longdiff = $longmax - $longmin; if( ( $latspacingspec * 1 ) <= 0 || ( $longspacingspec * 1 ) <= 0 ) { $haderror = true; print '

ERROR: header on line $linenumber does not supply valid (non-zero) values for latitude or longitude spacing, got; ' . htmlspecialchars($latspacing) . ' and ' . htmlspecialchars($longspacing) . '

'; break; } //OSI data has a rounding precision problem that causes it to miss its end point; 0.013333 is not the same as 5 / 375 //therefore this code tries to work out the actual spacing, by seeing how many data points can fit with approximately that spacing //and then adjusting the spacing accordingly if( $latdiff ) { $latcount = round( $latdiff / $latspacingspec ); $latspacing = $latdiff / $latcount; //fencepost problem $latcount++; } else { $latcount = 1; } if( $longdiff ) { $longcount = round( $longdiff / $longspacingspec ); $longspacing = $longdiff / $longcount; //fencepost problem $longcount++; } else { $longcount = 1; } $headerread = true; } elseif( $headerread ) { //every line after the header should be a series of numbers //usually they are grouped into blocks of several lines containing 8 data point values, with blank lines separating //groups which add up to a single line of latitude //the exact number does not matter, as long as the file has the right number of data points overall $datapoints = preg_split( "/\s+/", trim($line) ); foreach( $datapoints as $onenumber ) { if( !preg_match( "/^(-?\d\.?|-?\d*\.\d+)$/", $onenumber ) ) { $haderror = true; print "

ERROR: non-numeric data found where numeric data was expected on line $linenumber; " . htmlspecialchars($onenumber) . '

'; break 2; } $latindex = floor( $datacount / $longcount ); $longindex = $datacount % $longcount; $latitude = $latmax - $latindex * $latspacing; $longitude = $longmin + $longindex * $longspacing; if( !( $longindex ) ) { $alldata[$latindex] = array(); } $onenumber *= 1; //storing the data in an array uses a load of memory, but this is what allows it to only process the data when it has parsed correctly //it would be possible to send it to the browser for rendering at this point, but then a parsing error would result in a script error if( $_POST['dataaction'] != 'addlowmem' ) { $alldata[$latindex][$longindex] = ( $_POST['dataaction'] == 'render' || $_POST['dataaction'] == 'renderaddlowmem' ) ? $onenumber : array( $latitude, $longitude, $onenumber ); } if( $_POST['dataaction'] == 'renderaddlowmem' || $_POST['dataaction'] == 'addlowmem' ) { put_data_in_database( $longcount * $latindex + $longindex, $latindex, $longindex, $latitude, $longitude, $latstepdp, $longstepdp, $onenumber, $_POST['grdname'] ); } if( $onenumber > $highest ) { $highest = $onenumber; } if( $onenumber < $lowest ) { $lowest = $onenumber; } $datacount++; } } else { $haderror = true; print "

ERROR: file did not start with a header on line $linenumber, expected 6 numbers separated by spaces, got; " . htmlspecialchars($line) . '

'; break; } } } fclose($filehandle); if( !$haderror ) { if( !$headerread ) { $haderror = true; print '

ERROR: got to the end of the file, and did not discover a header.

'; } elseif( $datacount != $latcount * $longcount ) { $haderror = true; print '

ERROR: expected ' . ( $latcount * $longcount ) . " data points but got $datacount.

"; } } } else { $haderror = true; print '

ERROR: could not open file ' . htmlspecialchars($filename) . '

'; } $endtime = microtime(true); if( !$haderror ) { if( $_POST['dataaction'] == 'renderadd' || $_POST['dataaction'] == 'add' ) { /* THIS IS WHERE YOU CONNECT TO THE DATABASE $dbhandle = your database connection handle */ $dbhandle = @database_connect_command( 'somehost', 'someuser', 'somepassword', 'somedatabase' ); if( !$dbhandle ) { print '

ERROR: could not open database

'; fclose($filehandle); exit; } database_query_command( 'CREATE TABLE `'.addslashes($_POST['grdname']).'` ( recordindex int(10), latindex INT(5), longindex INT(5), latitude FLOAT, longitude FLOAT, shift FLOAT, PRIMARY KEY( recordindex ) ) CHARSET=utf8', $dbhandle ); if( $error = database_error_command() ) { print '

Error: '.htmlspecialchars($error).'

'; fclose($filehandle); exit; } for( $latindex = 0; $latindex < $latcount; $latindex++ ) { for( $longindex = 0; $longindex < $longcount; $longindex++ ) { put_data_in_database( $longcount * $latindex + $longindex, $latindex, $longindex, $alldata[$latindex][$longindex][0], $alldata[$latindex][$longindex][1], $latstepdp, $longstepdp, $alldata[$latindex][$longindex][2], $_POST['grdname'] ); } } //process any remaining batches put_data_in_database( null, null, null, null, null, null, null, null, null, true ); } elseif( $_POST['dataaction'] == 'renderaddlowmem' || $_POST['dataaction'] == 'addlowmem' ) { put_data_in_database( null, null, null, null, null, null, null, null, null, true ); } if( $_POST['dataaction'] == 'render' || $_POST['dataaction'] == 'renderaddlowmem' || $_POST['dataaction'] == 'renderadd' ) { //prepare to render the data using HTML canvas $datarange = $highest - $lowest; if( !$datarange ) { //avoid dividing by 0 $datarange = 0.000001; } print ''; ob_flush(); flush(); //build data[latindex][data,data,...,data] arrays print '

"; } print "
Latitude min
$latmin
Latitude max
$latmax
Longitude min
$longmin
Longitude max
$longmax
Latitude spacing
$latspacingspec
Longitude spacing
$longspacingspec
Latitude spacing decimal places
$latstepdp
Longitude spacing decimal places
$longstepdp
"; $imagetime = microtime(true); print "

Highest point discovered $highest, lowest point discovered $lowest. File contained $datacount points in $latcount rows of $longcount columns. Processing $filebytes bytes took " . ( $endtime - $starttime ) . ' seconds and ' . ( memory_get_usage() / ( 1024 * 1024 ) ) . 'MB of memory. HTML output took ' . ( $imagetime - $endtime ) . ' seconds.

'; } } elseif( isset($_FILES['grdfile']) ) { //handle file upload errors switch($_FILES['grdfile']['error']) { case UPLOAD_ERR_INI_SIZE: print '

ERROR: Uploaded file was too large. Max size ' . htmlspecialchars($maxsizereadable) . '.

'; break; case UPLOAD_ERR_PARTIAL: print '

ERROR: Uploaded file was partially incomplete; upload was probably interrupted.

'; break; case UPLOAD_ERR_NO_FILE: print '

ERROR: No file was uploaded.

'; break; case UPLOAD_ERR_NO_TMP_DIR: print '

ERROR: Server configuration fault; no upload temporary directory was specified.

'; break; case UPLOAD_ERR_CANT_WRITE: print '

ERROR: Server fault; could not write to disk - disk full?

'; break; case UPLOAD_ERR_EXTENSION: print '

ERROR: Server configuration fault; a PHP extension halted the upload.

'; break; default: print '

ERROR: Unknown error type.

'; } } $maxsize = strtolower( $maxsizereadable ); $chars = preg_replace( "/[^a-z]/i", '', $maxsize ); $maxsize = intval($maxsize); switch( $chars ) { case 't': $maxsize *= 1024; case 'g': $maxsize *= 1024; case 'm': $maxsize *= 1024; case 'k': $maxsize *= 1024; default: $maxsize *= 1; } ?>


WARNING: if you add to the database during parsing, and parsing fails part way through, the database will contain incomplete data.