If we use the basic approach of read.csv, we end up waiting a really long time,
system.time(park<-read.csv("/home/vis/cr173/Sta523/data/parking/NYParkingViolations.csv"))
## user system elapsed
## 544.698 7.411 555.405 Nearly 10 minutes to read in a 1.7 gigabyte CSV file.
system.time(read.csv("/home/vis/cr173/Sta523/data/parking/NYParkingViolations.csv",
stringsAsFactors=FALSE)
)
## user system elapsed
## 480.018 6.643 486.517 system.time(read.csv("/home/vis/cr173/Sta523/data/parking/NYParkingViolations.csv",
stringsAsFactors=FALSE,
comment.char="")
)
## user system elapsed
## 348.993 4.488 353.381 system.time(read.csv("/home/vis/cr173/Sta523/data/parking/NYParkingViolations.csv",
stringsAsFactors=FALSE,
comment.char="",
colClasses=c("numeric", "character", "character", "character", "character",
"integer", "character", "character", "character", "integer",
"integer", "integer", "integer", "integer", "integer",
"integer", "integer", "character", "character", "character",
"character", "character", "character", "character", "character",
"character", "integer", "integer", "character", "character",
"character", "character", "character", "character", "integer",
"integer", "character", "integer", "character", "character",
"logical", "logical", "logical")
)
)
## user system elapsed
## 239.846 2.583 242.328 library(data.table)
park = fread("/home/vis/cr173/Sta523/data/parking/NYParkingViolations.csv",sep=",")
## Read 9100278 rows and 43 (of 43) columns from 1.673 GB file in 00:00:44
## Warning messages:
## 1: In fread("/home/vis/cr173/Sta523/data/parking/NYParkingViolations.csv", :
## Bumped column 26 to type character on data row 15, field contains 'GOUVERNUR ST EAST RI'.
## Coercing previously read values in this column from logical, integer or numeric back to
## character which may not be lossless; e.g., if '00' and '000' occurred before they will now
## be just '0', and there may be inconsistencies with treatment of ',,' and ',NA,' too (if they
## occurred in this column before the bump). If this matters please rerun and set 'colClasses'
## to 'character' for this column. Please note that column type detection uses the first 5 rows,
## the middle 5 rows and the last 5 rows, so hopefully this message should be very rare. If
## reporting to datatable-help, please rerun and include the output from verbose=TRUE.## 2: In fread("/home/vis/cr173/Sta523/data/parking/NYParkingViolations.csv", :
## Bumped column 30 to type character on data row 2647, field contains 'T'. Coercing previously
## read values in this column from logical, integer or numeric back to character which may not be
## lossless; e.g., if '00' and '000' occurred before they will now be just '0', and there may be
## inconsistencies with treatment of ',,' and ',NA,' too (if they occurred in this column before the
## bump). If this matters please rerun and set 'colClasses' to 'character' for this column. Please
## note that column type detection uses the first 5 rows, the middle 5 rows and the last 5 rows, so
## hopefully this message should be very rare. If reporting to datatable-help, please rerun and
## include the output from verbose=TRUE.
## 3: In fread("/home/vis/cr173/Sta523/data/parking/NYParkingViolations.csv", :
## Some columns have been read as type 'integer64' but package bit64 isn't loaded. Those columns will
## display as strange looking floating point data. There is no need to reload the data. Just require(bit64)
## to obtain the integer64 print method and print the data again.class(park)
## "data.table" "data.frame"
park = as.data.frame(park)
class(park)
## [1] "data.frame"head(park)
## Summons Number Plate ID Registration State Plate Type Issue Date Violation Code
## 1 1361929741 FCJ5493 NY PAS 12/18/1970 20
## 2 1366962000 63540MC NY COM 02/02/1971 46
## 3 1356906515 GFM1421 NY PAS 09/18/1971 40
## 4 1342296217 FYM5117 NY SRF 09/18/1971 21
## 5 1342296199 95V6675 TX PAS 09/18/1971 21
## 6 1342296187 GCY4187 NY SRF 09/18/1971 21
##
## Vehicle Body Type Vehicle Make Issuing Agency Street Code1 Street Code2 Street Code3
## 1 SUBN GMC S 35030 89180 32600
## 2 DELV FRUEH P 58830 11430 11450
## 3 SDN MAZDA X 13610 37270 37290
## 4 SUBN NISSA S 28190 55690 11590
## 5 GMC S 11790 35780 57890
## 6 VAN FORD S 11790 35780 57890
##
## Vehicle Expiration Date Violation Location Violation Precinct Issuer Precinct Issuer Code
## 1 20140728 70 70 0 829379
## 2 0 68 68 68 928157
## 3 20150301 33 33 33 938472
## 4 20140427 104 104 0 515350
## 5 0 104 104 0 515350
## 6 20150430 104 104 0 515350
##
## ...We are working with data that is large, but will still fit in memory.
R loves to make extra copies of objects, so we need to be careful - even a handful of copies with exhaust the memory on most systems.
In general you should prefer:
subsetting >>>> apply > loops
library(lubridate)
names(park)
head(park[["Issue Date"]])
t = table(park[["Issue Date"]])
d = data.frame(date = mdy(names(t)), count = c(t))
head(d)
plot(d$date, d$count)
d = d[order(d$date),]
plot(d$date, d$count,type='l')
start = mdy("8/1/2013")
end = mdy("6/28/2014")
with(d[d$date >= start & d$date <= end,], plot(date, count,type='l'))
with(d[d$date >= start-months(1) & d$date <= end+months(1),], plot(date, count,type='l'))
d$wday = wday(d$date,label = TRUE)
head(d)
d_sub = d[d$date >= start & d$date <= end,]
t = tapply(d_sub$count, d_sub$wday, sum)
plot(factor(names(t),levels=names(t),ordered=TRUE),t)# Vehicle year distirbution
t = table(park[["Vehicle Year"]])[-1]
plot(as.numeric(names(t)),t,type='l')
t = t[as.numeric(names(t))<=2014]
plot(as.numeric(names(t)),t,type='l')# Street words
t = c(park[["Street Name"]],park[["Intersecting Street"]])
t = str_trim(t)
t = t[t != ""]
t = unlist(str_split(t," "))
t = t[t != ""]
t = tolower(t)
head(table(t))
t = str_replace(t,"^`","")
t = t[t != ""]
t = t[t != "-"]
head(table(t))
head(table(t), n=200)
tt = sort(table(t),decreasing = TRUE)
head(tt)
head(tt, n=50)