資料清理要做的幾件事

1
2
3

#變數結構
str(data)
summary(data)

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21

#1資料清洗
#1-1.缺失值處理,例如null.na....
n <- length(which(is.na(data)))+length(which(is.null(data))) 
#各變數缺失值數量>95%可以直接刪除該變數.
#各變數缺失值數量<95%需考慮以哪種方式填補,除了手動檢查原始資料外,例如偏態變數以中位數填補,正態以平均數填補或以線性迴歸計算缺失值.
if(n!=0){
  seat <- c() # seat:含缺失值變數位置
  na <- c()  # na:變數缺失值數量>95%變數位置
  for (i in 1:length(data[1,])) {
    if(length(which(is.na(data[,i])))+length(which(is.null(data[,i])))!=0){
      seat <- c(seat,i)
    }
  }
  for (i in seat) {
    if((length(which(is.na(data[,i])))+length(which(is.null(data[,i]))))/length(data[,i])>0.95){
      na <- c(na,i)
    }
  }
  if(is.null(na)=='FALSE'){
    data <- data[ ,-na]}
}

1
2
3
4
5

#1-2.1 重複值處理
n <- length(which(duplicated(data)=='TRUE')) #重複值數量
if(n!=0){
  data <- data[!duplicated(data), ]
}

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44

#1-2.2 null.na值處理,直接刪除或以中位數或均值替補.
#方法1.直接刪除
#方法2.偏態以中位數替補,正態以平均數替補
#用直方圖與盒鬚圖辨別辨別分佈型態
par(mfrow=c(2,4))
for(i in 1:length(seat)){
  hist(data[,seat[i]],freq = TRUE,
       main = paste("Histogram of" , colnames(data)[seat[i]]),
       xlab = colnames(data)[seat[i]])
  boxplot(data[,seat[i]],
          main = colnames(data)[seat[i]],
          xlab = "time",outcol="red")
}

#缺失值替代
MAX <- function(df){
  if(length(which(is.na(df))!=0)){df <- df[-which(is.na(df))]}
  Q1 <- as.data.frame(quantile(df, 1 / 4))[1,1] 
  Q3 <- as.data.frame(quantile(df, 3 / 4))[1,1]
  IQR <- Q3-Q1
  MAX <- Q3+1.5*IQR
  return(MAX)
}
MIN <- function(df){
  if(length(which(is.na(df))!=0)){df <- df[-which(is.na(df))]}
  Q1 <- as.data.frame(quantile(df, 1 / 4))[1,1] 
  Q3 <- as.data.frame(quantile(df, 3 / 4))[1,1]
  IQR <- Q3-Q1
  MIN <- Q1-1.5*IQR
  return(MIN)
}

seat_1 <- c(which(is.null(data[,seat[1]])),which(is.na(data[,seat[1]])));#BQ中位數
seat_2 <- c(which(is.null(data[,seat[5]])),which(is.na(data[,seat[5]])));#EL均值

#BQ
for (i in 1:length(seat_1)) {
  data[seat_1[i],seat[1]] <- median(data[,seat[1]],na.rm = TRUE)
}

#EL
for (i in 1:length(seat_2)) {
  data[seat_5[i],seat[5]] <- mean(data[,seat[5]],na.rm = TRUE)
}

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28

#Correlation : 考慮適度刪除高度相關的變數.
par(mfrow=c(1,1))
correlation <- cor(data[,-c(1,41,58,59)]) #Id與離散型變數不能算Correlation
corrplot(correlation)

diagonal <- c()
for (i in 1:55) {
  diagonal <- c(diagonal,c(55*(i-1)+i))
}
seat <- setdiff(which(correlation>=0.7),diagonal)
I <-c()
for (i in 1:length(seat)) {
  if(55*seat[i]%/%55+(seat[i]%/%55+1)<=seat[i]&seat[i]<=55*(seat[i]%/%55+1)){I <-c(I,i)}
}
seat <- seat[I]
x <- c()
y <- c()
for (i in 1:length(seat)) {
  x <- c(x,seat[i]%%55)
  y <- c(y,seat[i]%/%55+1)
}
cor <- c()
for(i in 1:length(x)){cor <- c(cor,correlation[x[i],y[i]])}
cor <- data.frame(row = rownames(correlation[x,]),
           col = colnames(correlation[,y]),
           cor = cor)
cor <- cor[order(cor$cor,decreasing = TRUE),]
rownames(cor) <- 1:length(x);cor

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11

library(caret)
#特徵重要程度排序
# prepare training scheme
control <- trainControl(method="repeatedcv", number=10, repeats=3)
# train the model
model <- train(Alpha~., data=data[,-c(1,41,58)], method="lvq", preProcess="scale", trControl=control)
# estimate variable importance
importance <- varImp(model, scale=FALSE)
# summarize importance
print(importance)
plot(importance,10)