basic firebrand recognition

Co-authored-by: Alex <AlexanderMcDowell@users.noreply.github.com>

examples/pyrometry/edge_detection.py

@@ -2,26 +2,77 @@
 
 import cv2 as cv
 import numpy as np
+from skimage import measure, morphology, color, segmentation
+import matplotlib.pyplot as plt
 
-# edge-detection kernel amplification
+file = 'streaktest2.png'
-AMPLIFIER=8
-
-MIN_INTENSITY=100
-
-# file = '01-0001-cropped.png'
-file = 'streaktest.png'
-file_name = file.split(".")[0]
-file_ext = file.split(".")[1]
-
 img = cv.imread(file)
 
-img = cv.cvtColor(img, cv.COLOR_BGR2GRAY)
+# blurred = cv.GaussianBlur(img, (8, 8), 0)
 
-kernel = np.array([
+retval, thresh_gray = cv.threshold(img, 120, 255, cv.THRESH_BINARY)
-    [-1, -1, -1],
-    [-1, AMPLIFIER, -1],
-    [-1, -1, -1],
-])
-img = cv.filter2D(src=img, ddepth=-1, kernel=kernel)
 
-cv.imwrite(f'{file_name}-edge-detection.{file_ext}', img)
+kernel = np.ones((7, 7), np.uint8)
+image = cv.morphologyEx(thresh_gray, cv.MORPH_CLOSE, kernel, iterations=1)
+gray = cv.cvtColor(image, cv.COLOR_BGR2GRAY)
+
+retval, gray = cv.threshold(gray, 0, 255, cv.THRESH_BINARY)
+
+gray = cv.copyMakeBorder(
+    gray, 
+    20, 
+    20, 
+    20, 
+    20, 
+    cv.BORDER_CONSTANT, 
+    value=0
+)
+
+# cv.imshow('gray', gray)
+# cv.waitKey(0)
+
+# contours = measure.find_contours(array=gray, level=100)
+_img, contours = cv.findContours(gray, cv.RETR_LIST, cv.CHAIN_APPROX_NONE)[0]
+
+fig, ax = plt.subplots()
+ax.imshow(gray, cmap=plt.cm.gray, alpha=1)
+
+def calculate_area(countour):
+    c = np.expand_dims(countour.astype(np.float32), 1)
+    c = cv.UMat(c)
+
+    return cv.contourArea(c)
+
+def center_of_mass(X):
+    x = X[:,0]
+    y = X[:,1]
+    g = (x[:-1]*y[1:] - x[1:]*y[:-1])
+    A = 0.5*g.sum()
+    cx = ((x[:-1] + x[1:])*g).sum()
+    cy = ((y[:-1] + y[1:])*g).sum()
+
+    return 1./(6*A)*np.array([cx,cy])
+
+
+img_new = cv.cvtColor(gray, cv.COLOR_GRAY2BGR)
+
+for contour in contours:
+    area = calculate_area(contour)
+
+    # if area > 250:
+        # cnt = np.array(contour).reshape((-1, 1, 2)).astype(np.int32)
+        # cv.drawContours(img_new, [cnt], -1, (0, 200, 255), thickness=10)
+
+    cv.drawContours(img_new, [contour], -1, (0, 200, 255), thickness=3)
+
+        # ax.plot(contour[:, 1], contour[:, 0], linewidth=0.5, color='orangered')
+
+# cv.imshow('contours', img_new)
+# cv.waitKey(0)
+
+cv.imwrite("firebrand_contours_opencv.png", img_new)
+
+ax.axis('image')
+ax.set_xticks([])
+ax.set_yticks([])
+plt.savefig("edge_detection_figure.png", dpi=500)

flask_frontend.py

@@ -1,11 +1,10 @@
 from flask import Flask, render_template, request, send_file
 import numpy as np
+from plotly_util import generate_plotly_temperature_pdf
 from ratio_pyrometry import ratio_pyrometry_pipeline
 from size_projection import get_projected_area
 import base64
 import cv2 as cv
-import plotly.figure_factory as ff
-import pandas as pd
 
 app = Flask(
     __name__, 
@@ -21,7 +20,7 @@ def index():
 def ratio_pyro():
     f = request.files['file']
     f_bytes = np.fromstring(f.read(), np.uint8)
-    img_orig, img_res, key, ptemps = ratio_pyrometry_pipeline(
+    img_orig, img_res, key, ptemps, indiv_firebrands = ratio_pyrometry_pipeline(
         f_bytes,
         ISO=float(request.form['iso']),
         I_Darkcurrent=float(request.form['i_darkcurrent']),
@@ -31,47 +30,28 @@ def ratio_pyro():
         MIN_TEMP=float(request.form['min_temp']),
         smoothing_radius=int(request.form['smoothing_radius']),
         key_entries=int(request.form['legend_entries']),
-        eqn_scaling_factor=float(request.form['equation_scaling_factor'])
+        eqn_scaling_factor=float(request.form['equation_scaling_factor']),
+        firebrand_min_intensity_threshold=float(request.form['intensity_threshold']),
+        firebrand_min_area=float(request.form['min_area']),
     )
 
     # get base64 encoded images
     img_orig_b64 = base64.b64encode(cv.imencode('.png', img_orig)[1]).decode(encoding='utf-8')
     img_res_b64 = base64.b64encode(cv.imencode('.png', img_res)[1]).decode(encoding='utf-8')
 
-    # generate prob. distribution histogram & return embed
+    ptemps_list = [ptemps]
-    fig = ff.create_distplot(
-        [ptemps], 
-        group_labels=[f.filename], 
-        show_rug=False,
-        show_hist=False,
-    )
-    fig.update_layout(
-        autosize=False,
-        width=800,
-        height=600,
-    )
-    fig.update_xaxes(
-        title_text="Temperature (°C)",
-    )
-    fig.update_yaxes(
-        title_text="Probability (1/°C)",
-    )
-    freq_plot = fig.to_html()
 
-    # create csv-formatted stuff
+    for i in range(len(indiv_firebrands)):
-    # currently only supports 1 firebrand (grabs first object in plot).
+        # base64 encode image data
-    plot_data=fig.to_dict()
+        brand_data = indiv_firebrands[i]
-    x_data = plot_data["data"][0]["x"]
+        unencoded = brand_data["img_data"]
-    y_data = plot_data["data"][0]["y"]
+        brand_data["img_data"] = base64.b64encode(cv.imencode('.png', unencoded)[1]).decode(encoding='utf-8')
+        indiv_firebrands[i] = brand_data
+        
+        # add ptemp data to list
+        ptemps_list.append(brand_data["ptemps"])
 
-    tdata = [["Temperature", "Frequency"]]
+    freq_plot, csvstrs = generate_plotly_temperature_pdf(ptemps_list)
-    for i in range(len(x_data)):
-        r = []
-        r.append(x_data[i])
-        r.append(y_data[i])
-        tdata.append(r)
-
-    csvstr = pd.DataFrame(tdata).to_csv(index=False, header=False)
 
     return render_template(
         'pyrometry-results.html',
@@ -79,7 +59,8 @@ def ratio_pyro():
         img_res_b64=img_res_b64,
         legend=key,
         freq_plot=freq_plot,
-        csv_data=csvstr
+        csv_data=csvstrs[0],
+        individual_firebrands=indiv_firebrands,
     )
 
 

plotly_util.py

@@ -0,0 +1,64 @@
+from typing import List
+import plotly.figure_factory as ff
+import pandas as pd
+
+def generate_plotly_temperature_pdf(ptemps_list: List[list]):
+    """
+    Generate plotly graph HTML & raw CSV data for temperature pdf
+
+    ptemps: pixel temperature LIST in order of:
+
+    - Ptemps of firebrands "overview" image
+    - Ptemps list for each individual firebrand
+
+    plotname: what to call the plot
+
+    Returns result in form (plot_html, csv_data)
+    """
+
+    # generate prob. distribution histogram & return embed
+    labels = ["Full Image"]
+    for i in range(len(ptemps_list[1:])):
+        labels.append(f"Firebrand {i+1}")
+    labels.reverse()
+
+    fig = ff.create_distplot(
+        ptemps_list,
+        group_labels=labels, 
+        show_rug=False,
+        show_hist=False,
+    )
+    fig.update_layout(
+        autosize=False,
+        width=800,
+        height=600,
+    )
+    fig.update_xaxes(
+        title_text="Temperature (°C)",
+    )
+    fig.update_yaxes(
+        title_text="Probability (1/°C)",
+    )
+    freq_plot = fig.to_html()
+
+    # create csv-formatted stuff
+    csvstrs = []
+    plot_data=fig.to_dict()
+    for i in range(len(plot_data["data"])):
+        x_data = plot_data["data"][i]["x"]
+        y_data = plot_data["data"][i]["y"]
+
+        tdata = [["Temperature", "Frequency"]]
+        for i in range(len(x_data)):
+            r = []
+            r.append(x_data[i])
+            r.append(y_data[i])
+            tdata.append(r)
+
+        csvstr = pd.DataFrame(tdata).to_csv(index=False, header=False)
+        csvstrs.append(csvstr)
+
+    return (
+        freq_plot,
+        csvstrs
+    )

ratio_pyrometry.py

@@ -3,6 +3,7 @@ from multiprocessing.sharedctypes import Value
 import cv2 as cv
 import numpy as np
 from numba import jit
+from skimage import measure
 
 @jit(nopython=True)
 def rg_ratio_normalize(
@@ -85,11 +86,104 @@ def ratio_pyrometry_pipeline(
     smoothing_radius: int,
     key_entries: int,
     eqn_scaling_factor: float,
+    # firebrand detection
+    firebrand_min_intensity_threshold: float,
+    firebrand_min_area: float
 ):
-
     # read image & crop
     img_orig = cv.imdecode(file_bytes, cv.IMREAD_UNCHANGED)
 
+    # ---------------------------------------------------------
+    # -- Firebrand detection
+    # ---------------------------------------------------------
+
+    img = cv.copyMakeBorder(
+        img_orig, 
+        20, 
+        20, 
+        20, 
+        20, 
+        cv.BORDER_CONSTANT, 
+        value=0
+    )
+
+    retval, thresh_gray = cv.threshold(img, firebrand_min_intensity_threshold, 255, cv.THRESH_BINARY)
+
+    kernel = np.ones((7, 7), np.uint8)
+    image = cv.morphologyEx(thresh_gray, cv.MORPH_CLOSE, kernel, iterations=1)
+    gray = cv.cvtColor(image, cv.COLOR_BGR2GRAY)
+
+    retval, gray = cv.threshold(gray, 0, 255, cv.THRESH_BINARY)
+
+    contours = measure.find_contours(array=gray, level=100)
+
+    def calculate_area(countour):
+        c = np.expand_dims(countour.astype(np.float32), 1)
+        c = cv.UMat(c)
+        
+        return cv.contourArea(c)
+
+    individual_firebrands = []
+
+    for contour in contours:
+        if calculate_area(contour) > firebrand_min_area:
+            mask = np.zeros(img.shape[0:2], dtype='uint8')
+            cv.fillPoly(mask, pts=np.int32([np.flip(contour, 1)]), color=(255,255,255))
+
+            retval, mask = cv.threshold(mask, 0, 255, cv.THRESH_BINARY)
+
+            #apply the mask to the img
+            masked = cv.bitwise_and(img, img, mask=mask)
+
+            masked_ratio_rg, ptemps_indiv = rg_ratio_normalize(
+                masked,
+                I_Darkcurrent,
+                f_stop,
+                exposure_time,
+                ISO,
+                MIN_TEMP,
+                MAX_TEMP,
+                eqn_scaling_factor,
+            )
+
+            # build & apply smoothing conv kernel
+            k = []
+            for i in range(smoothing_radius):
+                k.append([1/(smoothing_radius**2) for i in range(smoothing_radius)])
+            kernel = np.array(k)
+
+            masked_ratio_rg = cv.filter2D(src=masked_ratio_rg, ddepth=-1, kernel=kernel)
+
+            # write colormapped image
+            masked_ratio_rg_jet = cv.applyColorMap(masked_ratio_rg, cv.COLORMAP_JET)
+
+            # Generate key
+            step = (MAX_TEMP - MIN_TEMP) / (key_entries-1)
+            temps = []
+            key_img_arr = [[]]
+
+            for i in range(key_entries):
+                res_temp = MIN_TEMP + (i * step)
+                res_color = scale_temp(res_temp, MIN_TEMP, MAX_TEMP)
+                temps.append(math.floor(res_temp))
+                key_img_arr[0].append([res_color, res_color, res_color])
+
+            key_img = np.array(key_img_arr).astype(np.uint8)
+            key_img_jet = cv.applyColorMap(key_img, cv.COLORMAP_JET)
+
+            tempkey = {}
+
+            for i in range(len(temps)):
+                c = key_img_jet[0][i]
+                tempkey[temps[i]] = f"rgb({c[2]}, {c[1]}, {c[0]})"
+
+            individual_firebrands.append({
+                "img_data": masked_ratio_rg_jet,
+                "legend": tempkey,
+                "ptemps": ptemps_indiv
+            })
+
+
     img, ptemps = rg_ratio_normalize(
         img_orig,
         I_Darkcurrent,
@@ -112,7 +206,9 @@ def ratio_pyrometry_pipeline(
     # write colormapped image
     img_jet = cv.applyColorMap(img, cv.COLORMAP_JET)
 
-    # --- Generate temperature key ---
+    # ---------------------------------------------------------
+    # -- Generate temperature key
+    # ---------------------------------------------------------
 
     # Generate key
     step = (MAX_TEMP - MIN_TEMP) / (key_entries-1)
@@ -133,4 +229,4 @@ def ratio_pyrometry_pipeline(
         tempkey[temps[i]] = f"rgb({c[2]}, {c[1]}, {c[0]})"
 
     # original, transformed, legend
-    return img_orig, img_jet, tempkey, ptemps
+    return img_orig, img_jet, tempkey, ptemps, individual_firebrands

templates/index.html

@@ -50,8 +50,13 @@
 
     <h4>Firebrand Detection Settings</h4>
 
-    <label for="edge_amp">Edge Amplification</label>
+    <label for="intensity_threshold">intensity threshold (0-255)</label>
-    <input type="number" name="edge_amp" value="10", step="0.1"/>
+    <input type="number" name="intensity_threshold" value="250", step="0.1"/>
+    <br>
+
+    <label for="min_area">min area (removes small particles)</label>
+    <input type="number" name="min_area" value="115", step="0.1"/>
+    <br>
 
     <h4>Output Settings</h4>
 

templates/pyrometry-results.html

@@ -7,6 +7,9 @@
 {% block content %} 
 
 <div style="display:flex; flex-direction: column;">
+
+<h2>General Results</h2>
+
 <table class="img-table">
     <tr>
         <th class="img-table-heading">Input Image</th>
@@ -40,6 +43,44 @@
     </tr>
 </table>
 
+
+<h2>Individual Firebrands</h2>
+
+<table>
+    <tr>
+        <th>Output Heatmap</th>
+        <th>Legend</th>
+    </tr>
+
+    {% for item in individual_firebrands %}
+    <tr>
+        {# output heatmap #}
+        <td>
+            <img class="img-out" src="data:image/png;base64,{{ item['img_data'] }}" alt="result image">
+        </td>
+
+        </td>
+        {# legend #}
+        <td>
+            <h3>Firebrand {{ loop.index }}</h3>
+            <table class="legend" id="legend">
+                <tr>
+                    <th class="legend-heading">Color</th>
+                    <th class="legend-heading">Temperature</th>
+                </tr>
+                {% for temp, color in item["legend"].items() %}
+                <tr>
+                    <td class="legend-cell"><div style="width:30px;height:20px;background-color:{{ color }};"></div></td>
+                    <td class="legend-cell">{{ temp }}°C</td>
+                </tr>
+                {% endfor %}
+            </table>
+        </td>
+    </tr>
+    {% endfor %}
+
+</table>
+
 <br>
 {# Temperature Frequency Plot #}
 <div style="display: flex; flex-direction: row; align-items: center;">
@@ -50,6 +91,9 @@ onclick="saveData(`{{csv_data}}`, 'temperature-data.csv')">Download Data as CSV<
 </div>
 {{ freq_plot | safe }}
 
+
+<!-- Firebrands: {{ individual_firebrands }} -->
+
 </div>
 
 {% endblock %}