ping packet loss updates

docker/pinger/BMping.sh

+#!/bin/bash
+
+endpoints="192.168.111.200"
+filename="/home/llh/pingStats/pingStats.json"
+downtime=1
+gpstopic="/rossettiModem/dji_sdk/gps_position"
+
+echo "pinging: $endpoints"
+echo "save to: $filename"
+echo "gps topic: $gpstopic"
+echo " "
+
+# enable ros commands
+source /opt/ros/noetic/setup.bash
+source /home/larkin/catkin_ws/devel/setup.bash
+
+ROS_IP=192.168.111.202
+ROS_MASTER_URI=http://192.168.111.200:11311
+
+echo ping "${endpoints} every ${downtime:=1} sec, saving to ${filename}"
+# for endpoints in "${endpoints[@]}"; do
+# done
+while true; do
+
+  rawData=$(ping -c 3 -q -w 3 ${endpoints:=localhost})
+  newPacketData=$(echo "$rawData" | grep -oP '\d+(?=% packet loss)')
+  newLatencyData=$(echo "$rawData" | tail -n 1)
+
+  OLDIFS=$IFS
+  IFS=" "
+  read bit1 bit2 bit3 bit4 bit5 <<< "$newLatencyData"
+  IFS="/"
+  read num1 num2 num3 num4 <<< "$bit4"
+  IFS=$OLDIFS
+
+  echo "------------------"
+  if [[ -z $num4 ]]; then
+    # no network data, or all packets lost
+    num1="0"
+    num2="0"
+    num3="0"
+    num4="0"
+  fi
+
+  # debugging help
+  # echo "------------------"
+  # echo $num1
+  # echo $num2
+  # echo $num3
+  # echo $num4
+  # echo "------------------"
+  # echo "$rawData"
+  # echo "------------------"
+  # echo "$newPacketData"
+  # echo "------------------"
+  # echo "$newLatencyData"
+  # exit
+
+  if test -f "$filename"; then
+    false
+  else
+    touch "$filename"
+    echo "{}" > "$filename"
+  fi
+
+  # get gps data if present (timeout 5s)
+  gpsData=$(timeout 5 rostopic echo -n 1 $gpstopic | grep "latitude:\|longitude:\|altitude:")
+  gpsData=$(echo $gpsData | tr '\n' ' ')
+  OLDIFS=$IFS
+  IFS=" "
+  read trash1 latitude trash2 longitude trash3 altitude <<< "$gpsData"
+  IFS=$OLDIFS
+
+  # ping can return standard deviation and gps data may not be available
+  if [[ -z "$latitude" ]]; then
+    # echo "gps data is not there"
+    jq --arg timedate $(date +"%H:%M_%F") --arg target $endpoints --argjson packetLoss $newPacketData --argjson rttMin $num1 --argjson rttAvg $num2 --argjson rttMax $num3 --argjson stdDev $num4 '.[$target] += [{"min":$rttMin,"avg":$rttAvg,"max":$rttMax,"std":$stdDev,"timedate":$timedate,"packetLoss":$packetLoss}]' "$filename" > "$filename.tmp" && mv "$filename.tmp" "$filename"
+  else
+    echo "gps data IS there"
+    echo $latitude
+    echo $longitude
+    echo $altitude
+    jq --arg timedate $(date +"%H:%M_%F") --arg target $endpoints --argjson altitude $altitude --argjson longitude $longitude --argjson latitude $latitude --argjson packetLoss $newPacketData --argjson rttMin $num1 --argjson rttAvg $num2 --argjson rttMax $num3 --argjson stdDev $num4 '.[$target] += [{"lat":$latitude,"long":$longitude,"alt":$altitude,"min":$rttMin,"avg":$rttAvg,"max":$rttMax,"std":$stdDev,"timedate":$timedate,"packetLoss":$packetLoss}]' "$filename" > "$filename.tmp" && mv "$filename.tmp" "$filename"
+  fi
+  sleep ${downtime:=1}
+done
+
+if test -f "$filename.tmp"; then
+  rm "$filename.tmp"
+fi

docker/pinger/Dping.sh

+#!/bin/bash
+docker run --env endpoints=192.168.111.200 --env filename=/tmp/pingStats/pingStatsDocker.json --env gpstopic=/rossettiModem/dji_sdk/gps_position --env downtime=1 --env ROS_IP=192.168.111.202 --env ROS_MASTER_URI=http://192.168.111.200:11311 -v /home/larkin/pingStats:/tmp/pingStats/ doorbell:latest /ping.sh

docker/pinger/ping.sh

@@ -21,32 +21,45 @@ fi
 # enable ros commands
 source /ros_entrypoint.sh
 
-echo ping "${endpoints} every ${downtime:=3} sec, saving to ${filename}"
+echo ping "${endpoints} every ${downtime:=1} sec, saving to ${filename}"
 # for endpoints in "${endpoints[@]}"; do
 # done
 while true; do
 
-  # for endpoints in "${endpoints[@]}"; do
-
-  # echo pinging "${endpoints} now ..."
-  newData=$(ping -c 3 ${endpoints:=localhost} | tail -n 1)
-
-  # if nothing comes back, we dont write anything
-  if [[ -n "$newData" ]]; then
-    false
-    echo "$newData"
-  else
-    echo "${endpoints} might be unreachable"
-    continue
-  fi
+  rawData=$(ping -c 3 -q -w 3 ${endpoints:=localhost})
+  newPacketData=$(echo "$rawData" | grep -oP '\d+(?=% packet loss)')
+  newLatencyData=$(echo "$rawData" | tail -n 1)
 
   OLDIFS=$IFS
   IFS=" "
-  read bit1 bit2 bit3 bit4 bit5 <<< "$newData"
+  read bit1 bit2 bit3 bit4 bit5 <<< "$newLatencyData"
   IFS="/"
   read num1 num2 num3 num4 <<< "$bit4"
   IFS=$OLDIFS
 
+  echo "------------------"
+  if [[ -z $num4 ]]; then
+    # no network data, or all packets lost
+    num1="0"
+    num2="0"
+    num3="0"
+    num4="0"
+  fi
+
+  # debugging help
+  # echo "------------------"
+  # echo $num1
+  # echo $num2
+  # echo $num3
+  # echo $num4
+  # echo "------------------"
+  # echo "$rawData"
+  # echo "------------------"
+  # echo "$newPacketData"
+  # echo "------------------"
+  # echo "$newLatencyData"
+  # exit
+
   if test -f "$filename"; then
     false
   else
@@ -62,37 +75,20 @@ while true; do
   read trash1 latitude trash2 longitude trash3 altitude <<< "$gpsData"
   IFS=$OLDIFS
 
-  echo $latitude
-  echo $longitude
-  echo $altitude
-  echo "------------------"
-
   # ping can return standard deviation and gps data may not be available
   if [[ -z "$latitude" ]]; then
     # echo "gps data is not there"
-    if [[ -n "$num4" ]]; then
-      jq --arg timedate $(date +"%H:%M_%F") --arg target $endpoints --argjson rttMin $num1 --argjson rttAvg $num2 --argjson rttMax $num3 --argjson stdDev $num4 '.[$target][$timedate] += [{"min":$rttMin,"avg":$rttAvg,"max":$rttMax,"std":$stdDev}]' "$filename" > "$filename.tmp" && mv "$filename.tmp" "$filename"
-    else
-      jq --arg timedate $(date +"%H:%M_%F") --arg target $endpoints --argjson rttMin $num1 --argjson rttAvg $num2 --argjson rttMax $num3 '.[$target][$timedate] += [{"min":$rttMin,"avg":$rttAvg,"max":$rttMax}]' "$filename" > "$filename.tmp" && mv "$filename.tmp" "$filename"
-    fi
+    jq --arg timedate $(date +"%H:%M_%F") --arg target $endpoints --argjson packetLoss $newPacketData --argjson rttMin $num1 --argjson rttAvg $num2 --argjson rttMax $num3 --argjson stdDev $num4 '.[$target] += [{"min":$rttMin,"avg":$rttAvg,"max":$rttMax,"std":$stdDev,"timedate":$timedate,"packetLoss":$packetLoss}]' "$filename" > "$filename.tmp" && mv "$filename.tmp" "$filename"
   else
     echo "gps data IS there"
     echo $latitude
     echo $longitude
     echo $altitude
-    if [[ -n "$num4" ]]; then
-      jq --arg timedate $(date +"%H:%M_%F") --arg target $endpoints --argjson altitude $altitude --argjson longitude $longitude --argjson latitude $latitude --argjson rttMin $num1 --argjson rttAvg $num2 --argjson rttMax $num3 --argjson stdDev $num4 '.[$target][$timedate] += [{"lat":$latitude,"long":$longitude,"alt":$altitude,"min":$rttMin,"avg":$rttAvg,"max":$rttMax,"std":$stdDev}]' "$filename" > "$filename.tmp" && mv "$filename.tmp" "$filename"
-    else
-      jq --arg timedate $(date +"%H:%M_%F") --arg target $endpoints --argjson altitude $altitude --argjson longitude $longitude --argjson latitude $latitude --argjson rttMin $num1 --argjson rttAvg $num2 --argjson rttMax $num3 '.[$target][$timedate] += [{"lat":$latitude,"long":$longitude,"alt":$altitude,"min":$rttMin,"avg":$rttAvg,"max":$rttMax}]' "$filename" > "$filename.tmp" && mv "$filename.tmp" "$filename"
+    jq --arg timedate $(date +"%H:%M_%F") --arg target $endpoints --argjson altitude $altitude --argjson longitude $longitude --argjson latitude $latitude --argjson packetLoss $newPacketData --argjson rttMin $num1 --argjson rttAvg $num2 --argjson rttMax $num3 --argjson stdDev $num4 '.[$target] += [{"lat":$latitude,"long":$longitude,"alt":$altitude,"min":$rttMin,"avg":$rttAvg,"max":$rttMax,"std":$stdDev,"timedate":$timedate,"packetLoss":$packetLoss}]' "$filename" > "$filename.tmp" && mv "$filename.tmp" "$filename"
   fi
-  fi
-  sleep ${downtime:=3}
+  sleep ${downtime:=1}
 done
 
 if test -f "$filename.tmp"; then
   rm "$filename.tmp"
 fi
-
-# if test -f "$nfsFilename.tmp"; then
-#   rm "$nfsFilename.tmp"
-# fi

plottingTools/mapPlotter.py

@@ -24,8 +24,17 @@ def inflateBounds(bbox=[38.865,-77.072,38.876,-77.001], amt=0.001):
     return [bbox[0]-amt, bbox[1]-amt, bbox[2]+amt, bbox[3]+amt]
 
 
+font = {'family' : 'DejaVu Sans',
+        'weight' : 'normal',
+        'size'   : 8}
+matplotlib.rc('font', **font)
+
 filenames = ["/home/llh/ros-kubernetes/plottingTools/pingStats_may_29th_2022/pingStats.json",
              "/home/llh/ros-kubernetes/plottingTools/pingStats_may_29th_2022/pingStatsDocker.json"]
+# filenames = ["/home/llh/ros-kubernetes/plottingTools/pingStats_june_3rd_2022/pingStats.json",
+#              "/home/llh/ros-kubernetes/plottingTools/pingStats_june_3rd_2022/pingStatsDocker.json",
+#              "/home/llh/ros-kubernetes/plottingTools/pingStats_june_3rd_2022/pingStatsKubernetes.json"]
+plotColors = ["orange","slateblue","mediumspringgreen"]
 lats = []
 longs = []
 avgLatencies = []
@@ -71,16 +80,8 @@ ax.xaxis.set_major_formatter(FormatStrFormatter('%.2f'))
 
 mapPlot = plt.imread('mapImg.jpeg')
 for i in range(len(lats)):
-    # get latency coloring
-    if i == 0:
-        latencyColor = [((maxAvgLatency-val*0.7)/maxAvgLatency,0.0,0.0) for val in avgLatencies[i]]
-    if i == 1:
-        latencyColor = [(0.0,(maxAvgLatency-val*0.7)/maxAvgLatency,0.0) for val in avgLatencies[i]]
-    else:
-        latencyColor = [(0.0,0.0,(maxAvgLatency-val*0.7)/maxAvgLatency) for val in avgLatencies[i]]
-
-    latencySize = [20*(siz/maxAvgLatency) + 5 for siz in avgLatencies[i]]
-    plt.scatter(longs[i], lats[i], zorder=1, alpha=1, c=latencyColor, s=latencySize, label=[f.split('/') for f in filenames][i][-1])
+    latencySize = [50*(siz/maxAvgLatency) + 5 for siz in avgLatencies[i]]
+    plt.scatter(longs[i], lats[i], zorder=1, alpha=0.8, c=plotColors[i], s=latencySize, label=[f.split('/') for f in filenames][i][-1])
 
 plt.imshow(mapPlot, zorder=0, extent=[bigBox[1], bigBox[3], bigBox[0], bigBox[2]], aspect='equal')
 plt.title('Network latency visualized')

plottingTools/pingStats_june_3rd_2022/pingStatsKubernetes.json

 {
   "192.168.111.200": {
-    "15:17_2022-05-29": [
-      {
-        "min": 14.019,
-        "avg": 25.007,
-        "max": 46.818,
-        "std": 15.422
-      },
-      {
-        "min": 14.195,
-        "avg": 18.927,
-        "max": 27.236,
-        "std": 5.894
-      },
-      {
-        "min": 12.568,
-        "avg": 13.834,
-        "max": 14.585,
-        "std": 0.9
-      }
-    ],
-    "15:18_2022-05-29": [
-      {
-        "min": 13.465,
-        "avg": 124.664,
-        "max": 218.968,
-        "std": 84.742
-      },
-      {
-        "min": 13.566,
-        "avg": 13.816,
-        "max": 14.133,
-        "std": 0.236
-      },
-      {
-        "min": 13.581,
-        "avg": 13.776,
-        "max": 14.167,
-        "std": 0.276
-      }
-    ],
-    "15:19_2022-05-29": [
-      {
-        "lat": 37.19727999999999,
-        "long": -80.577791,
-        "alt": 100.0999984741211,
-        "min": 15.273,
-        "avg": 125.15,
-        "max": 235.027,
-        "std": 109.877
-      },
-      {
-        "lat": 37.19727999999999,
-        "long": -80.577791,
-        "alt": 100.0999984741211,
-        "min": 13.945,
-        "avg": 14.194,
-        "max": 14.352,
-        "std": 0.178
-      },
-      {
-        "lat": 37.19727999999999,
-        "long": -80.577791,
-        "alt": 100.0999984741211,
-        "min": 13.549,
-        "avg": 17.37,
-        "max": 24.161,
-        "std": 4.814
-      }
-    ],
     "14:11_2022-06-03": [
       {
         "min": 14.017,

plottingTools/scatterPlotter.py

+# api key from gurgle AIzaSyDDFrIPAMv3HS6PXTDcWfiUys8si_-yZ-c
+import numpy as np
+from gmplot import GoogleMapPlotter
+from random import random
+import json
+from pprint import PrettyPrinter
+import matplotlib.pyplot as plt
+import matplotlib
+from matplotlib.ticker import FormatStrFormatter
+import shutil
+import requests
+from geopy import distance
+
+def getMapImage(bbox=[38.865,-77.072,38.876,-77.001], apiKey="GKvj5HV6DLjqJT8awam4KQL3d26GP1M1", mapType="sat", mapSize=[600,400]):
+    req = "https://www.mapquestapi.com/staticmap/v5/map?key={}&boundingBox={}&type={}&size={}".format(apiKey,','.join(map(str,bbox)),mapType,','.join(map(str,mapSize)))
+    response = requests.get(req, stream=True)
+    with open('mapImg.jpeg', 'wb') as out_file:
+        shutil.copyfileobj(response.raw, out_file)
+    del response
+
+def inflateBounds(bbox=[38.865,-77.072,38.876,-77.001], amt=0.001):
+    return [bbox[0]-amt, bbox[1]-amt, bbox[2]+amt, bbox[3]+amt]
+
+def calculateDistance(pt1, pt2):
+    # 2D geodesic distance in meters
+    distance_2d = distance.distance(pt1[:2], pt2[:2]).m
+    # 3D euclidean distance
+    distance_3d = np.sqrt(distance_2d**2 + (pt2[2] - pt1[2])**2)
+    return distance_3d
+
+def rejectOutliers(data, m = 2.):
+    d = np.abs(data - np.median(data))
+    mdev = np.median(d)
+    s = d/mdev if mdev else 0.
+    # return data[s<m]
+    return s<m
+
+filenames = ["/home/llh/ros-kubernetes/plottingTools/pingStats_june_3rd_2022/pingStats.json",
+             "/home/llh/ros-kubernetes/plottingTools/pingStats_june_3rd_2022/pingStatsDocker.json",
+             "/home/llh/ros-kubernetes/plottingTools/pingStats_june_3rd_2022/pingStatsKubernetes.json"]
+# filenames = ["/home/llh/ros-kubernetes/plottingTools/pingStats_may_29th_2022/pingStats.json",
+#              "/home/llh/ros-kubernetes/plottingTools/pingStats_may_29th_2022/pingStatsDocker.json"]
+             # "/home/llh/ros-kubernetes/plottingTools/pingStats_may_29th_2022/pingStatsKubernetes.json"]
+plotColors = ["orange","slateblue","mediumspringgreen"]
+# reference point of the base station
+anchorPoint = [37.196927, -80.578399, 452.7027587890625]
+
+# potentially the least python-ish way to handle this
+for (i, fn) in enumerate(filenames):
+    with open(fn) as file:
+        data = json.load(file)
+
+    # need to iterate over endpoints as well
+    endpoint = "192.168.111.200"
+    # ignore time stamps for now
+    timelessData = []
+    for minuteData in data[endpoint].values():
+        for pt in minuteData:
+            if ("lat" in pt):
+                timelessData.append(pt)
+
+    dists = np.array([calculateDistance([dataPoint["lat"], dataPoint["long"], dataPoint["alt"]],anchorPoint) for dataPoint in timelessData])
+    avgLatencies = np.array([dataPoint["avg"] for dataPoint in timelessData])
+    maskArray = rejectOutliers(avgLatencies, m=2.0)
+    filteredLatencies = avgLatencies[maskArray]
+    filteredDistances = dists[maskArray]
+    print("removed {outliers} outliers from the dataset".format(outliers=np.size(maskArray) - np.count_nonzero(maskArray)))
+
+    # calculate best fit line
+    func = np.poly1d(np.polyfit(filteredDistances, filteredLatencies, 1))
+    evenDists = np.linspace(np.min(filteredDistances),np.max(filteredDistances),10)
+    plt.scatter(filteredDistances, filteredLatencies, s=12, c=plotColors[i], label=[f.split('/') for f in filenames][i][-1])
+    plt.plot(evenDists, func(evenDists), c=plotColors[i], linewidth=1.5)
+
+plt.ticklabel_format(style='plain') # to prevent scientific notation
+plt.ticklabel_format(useOffset=False)
+
+plt.title('Network Latency vs Distance')
+plt.xlabel("Distance")
+plt.ylabel("Latency")
+plt.legend()
+# plt.show()
+plt.savefig("scatterPlot.jpeg", dpi=400)