DL-For-RaspberryPis/test.py at main · pdr24/DL-For-RaspberryPis · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
import collect_data
import model_setup
import cifar10_setup

import sys
import torch
import torch.nn as nn
import torch.optim as optim
import torchvision
import pandas as pd
import numpy as np
import time
import os

def test(model, test_loader, device, system_metrics):
    model.eval()
    correct = 0
    total = 0
    with torch.no_grad():
        for images, labels in test_loader:
            images, labels = images.to(device), labels.to(device)

            start_time = time.time()
            outputs = model(images)
            end_time = time.time()

            _, predicted = torch.max(outputs, 1)
            total += labels.size(0)
            correct += (predicted == labels).sum().item()

            inference_time = end_time - start_time
            cpu_percent, load_1min, load_5min, load_15min, cpu_freq_current = collect_data.collect_cpu_statistics()
            available_memory, used_memory, percent_used, active_memory, inactive_memory, buffers, cached, shared, swap_used, swap_free, swap_percent = collect_data.collect_memory_statistics()

            system_metrics = collect_data.add_metrics_to_df(system_metrics, inference_time, cpu_percent, load_1min, load_5min, load_15min, cpu_freq_current, available_memory, used_memory, percent_used, active_memory, inactive_memory, buffers, cached, shared, swap_used, swap_free, swap_percent)


    # Compute accuracy
    accuracy = 100.0 * correct / total
    print(f'Accuracy: {accuracy:.2f}%')

    return system_metrics, accuracy

if __name__ == "__main__":
    # check if command line arguments were valid
    if len(sys.argv) != 5:
        print("Usage: python3 test.py [mobilenet|inception|resnet18|alexnet|vgg16|squeezenet] [path_to_saved_model] [path_to_save_results] [number_iterations]")
        sys.exit(1)
    elif sys.argv[1].lower() not in ["mobilenet", "inception", "resnet18", "alexnet", "vgg16", "squeezenet"]:
        print("Error: model name must be either 'mobilenet', 'inception', 'resnet18', 'alexnet', 'vgg16', 'squeezenet'")
        print("Usage: python3 test.py [mobilenet|inception|resnet18|alexnet|vgg16] [path_to_saved_model] [path_to_save_results] [number_iterations]")
        sys.exit(1)

    # determine model to run training on
    model_name = sys.argv[1]
    print(f"Model: {model_name}")

    # path the saved model weights are located at
    path = sys.argv[2]

    # path to file to save results to
    results_path = sys.argv[3]

    # number of iterations to test
    number_iterations = int(sys.argv[4])

    # set the device (GPU or CPU)
    device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
    print(f"Device: {device}\n")

    # load model specified by the user
    model = model_setup.load_saved_model(model_name, path, device)

    # setup data loaders (cifar-10)
    number_test_samples = 500 # number of samples to test on
    train_loader, test_loader = cifar10_setup.setup_data(model_name, number_test_samples)

    # create df to store data
    system_metrics = collect_data.setup_df()
    pd.set_option('display.max_columns', None)  # Show all columns

    if number_iterations == 1:
        # test
        system_metrics, accuracy = test(model, test_loader, device, system_metrics)

        # save summary of collected system metrics
        system_metrics.describe(include='all').to_csv(results_path)
    else:
        for i in range(number_iterations):
            # test
            system_metrics = collect_data.setup_df()
            system_metrics, accuracy = test(model, test_loader, device, system_metrics)

            # file name for specific iteration
            results_filename = os.path.basename(results_path)
            new_results_filename = str(i + 1) + "_" + results_filename
            directory_path = os.path.dirname(results_path)
            new_path = os.path.join(directory_path, new_results_filename)

             # save summary of collected system metrics
            system_metrics.describe(include='all').to_csv(new_path)