basic firebrand recognition

Co-authored-by: Alex <AlexanderMcDowell@users.noreply.github.com>

examples/pyrometry/edge_detection.py

@@ -2,26 +2,77 @@
 
 import cv2 as cv
 import numpy as np
+from skimage import measure, morphology, color, segmentation
+import matplotlib.pyplot as plt
 
-# edge-detection kernel amplification
-AMPLIFIER=9
-
-MIN_INTENSITY=100
-
-# file = '01-0001-cropped.png'
-file = 'streaktest.png'
-file_name = file.split(".")[0]
-file_ext = file.split(".")[1]
-
+file = 'streaktest2.png'
 img = cv.imread(file)
 
-img = cv.cvtColor(img, cv.COLOR_BGR2GRAY)
+# blurred = cv.GaussianBlur(img, (8, 8), 0)
 
-kernel = np.array([
-    [-1, -1, -1],
-    [-1, AMPLIFIER, -1],
-    [-1, -1, -1],
-])
-img = cv.filter2D(src=img, ddepth=-1, kernel=kernel)
+retval, thresh_gray = cv.threshold(img, 120, 255, cv.THRESH_BINARY)
 
-cv.imwrite(f'{file_name}-edge-detection.{file_ext}', img)
+kernel = np.ones((7, 7), np.uint8)
+image = cv.morphologyEx(thresh_gray, cv.MORPH_CLOSE, kernel, iterations=1)
+gray = cv.cvtColor(image, cv.COLOR_BGR2GRAY)
+
+retval, gray = cv.threshold(gray, 0, 255, cv.THRESH_BINARY)
+
+gray = cv.copyMakeBorder(
+    gray, 
+    20, 
+    20, 
+    20, 
+    20, 
+    cv.BORDER_CONSTANT, 
+    value=0
+)
+
+# cv.imshow('gray', gray)
+# cv.waitKey(0)
+
+# contours = measure.find_contours(array=gray, level=100)
+_img, contours = cv.findContours(gray, cv.RETR_LIST, cv.CHAIN_APPROX_NONE)[0]
+
+fig, ax = plt.subplots()
+ax.imshow(gray, cmap=plt.cm.gray, alpha=1)
+
+def calculate_area(countour):
+    c = np.expand_dims(countour.astype(np.float32), 1)
+    c = cv.UMat(c)
+
+    return cv.contourArea(c)
+
+def center_of_mass(X):
+    x = X[:,0]
+    y = X[:,1]
+    g = (x[:-1]*y[1:] - x[1:]*y[:-1])
+    A = 0.5*g.sum()
+    cx = ((x[:-1] + x[1:])*g).sum()
+    cy = ((y[:-1] + y[1:])*g).sum()
+
+    return 1./(6*A)*np.array([cx,cy])
+
+
+img_new = cv.cvtColor(gray, cv.COLOR_GRAY2BGR)
+
+for contour in contours:
+    area = calculate_area(contour)
+
+    # if area > 250:
+        # cnt = np.array(contour).reshape((-1, 1, 2)).astype(np.int32)
+        # cv.drawContours(img_new, [cnt], -1, (0, 200, 255), thickness=10)
+
+    cv.drawContours(img_new, [contour], -1, (0, 200, 255), thickness=3)
+
+        # ax.plot(contour[:, 1], contour[:, 0], linewidth=0.5, color='orangered')
+
+# cv.imshow('contours', img_new)
+# cv.waitKey(0)
+
+cv.imwrite("firebrand_contours_opencv.png", img_new)
+
+ax.axis('image')
+ax.set_xticks([])
+ax.set_yticks([])
+plt.savefig("edge_detection_figure.png", dpi=500)

flask_frontend.py

@@ -1,11 +1,10 @@
-from flask import Flask, render_template, request
+from flask import Flask, render_template, request, send_file
 import numpy as np
+from plotly_util import generate_plotly_temperature_pdf
 from ratio_pyrometry import ratio_pyrometry_pipeline
 from size_projection import get_projected_area
 import base64
 import cv2 as cv
-import plotly.figure_factory as ff
-from scipy import stats
 
 app = Flask(
     __name__, 
@@ -21,7 +20,7 @@ def index():
 def ratio_pyro():
     f = request.files['file']
     f_bytes = np.fromstring(f.read(), np.uint8)
-    img_orig, img_res, key, ptemps = ratio_pyrometry_pipeline(
+    img_orig, img_res, key, ptemps, indiv_firebrands = ratio_pyrometry_pipeline(
         f_bytes,
         ISO=float(request.form['iso']),
         I_Darkcurrent=float(request.form['i_darkcurrent']),
@@ -31,39 +30,37 @@ def ratio_pyro():
         MIN_TEMP=float(request.form['min_temp']),
         smoothing_radius=int(request.form['smoothing_radius']),
         key_entries=int(request.form['legend_entries']),
-        eqn_scaling_factor=float(request.form['equation_scaling_factor'])
+        eqn_scaling_factor=float(request.form['equation_scaling_factor']),
+        firebrand_min_intensity_threshold=float(request.form['intensity_threshold']),
+        firebrand_min_area=float(request.form['min_area']),
     )
 
     # get base64 encoded images
     img_orig_b64 = base64.b64encode(cv.imencode('.png', img_orig)[1]).decode(encoding='utf-8')
     img_res_b64 = base64.b64encode(cv.imencode('.png', img_res)[1]).decode(encoding='utf-8')
 
-    # generate prob. distribution histogram & return embed
-    fig = ff.create_distplot(
-        [ptemps], 
-        group_labels=[f.filename], 
-        show_rug=False,
-        show_hist=False,
-    )
-    fig.update_layout(
-        autosize=False,
-        width=800,
-        height=600,
-    )
-    fig.update_xaxes(
-        title_text="Temperature (°C)",
-    )
-    fig.update_yaxes(
-        title_text="Probability (1/°C)",
-    )
-    freq_plot = fig.to_html()
+    ptemps_list = [ptemps]
+
+    for i in range(len(indiv_firebrands)):
+        # base64 encode image data
+        brand_data = indiv_firebrands[i]
+        unencoded = brand_data["img_data"]
+        brand_data["img_data"] = base64.b64encode(cv.imencode('.png', unencoded)[1]).decode(encoding='utf-8')
+        indiv_firebrands[i] = brand_data
+        
+        # add ptemp data to list
+        ptemps_list.append(brand_data["ptemps"])
+
+    freq_plot, csvstrs = generate_plotly_temperature_pdf(ptemps_list)
 
     return render_template(
         'pyrometry-results.html',
         img_orig_b64=img_orig_b64,
         img_res_b64=img_res_b64,
         legend=key,
-        freq_plot=freq_plot
+        freq_plot=freq_plot,
+        csv_data=csvstrs[0],
+        individual_firebrands=indiv_firebrands,
     )
 
 
@@ -81,6 +78,8 @@ def projected_area_results():
         f_bytes,
         int(request.form['area_threshold']),
         int(request.form['min_display_threshold']),
+        float(request.form['paper_width']),
+        float(request.form['paper_width'])
     )
 
     return render_template(
@@ -88,3 +87,7 @@ def projected_area_results():
         img_b64=img,
         dtable=dtable
     )
+
+# @app.route("/download_pyrometry_temps")
+# def download_pyrometry_temps():
+#     return send_file()

plotly_util.py

@@ -0,0 +1,64 @@
+from typing import List
+import plotly.figure_factory as ff
+import pandas as pd
+
+def generate_plotly_temperature_pdf(ptemps_list: List[list]):
+    """
+    Generate plotly graph HTML & raw CSV data for temperature pdf
+
+    ptemps: pixel temperature LIST in order of:
+
+    - Ptemps of firebrands "overview" image
+    - Ptemps list for each individual firebrand
+
+    plotname: what to call the plot
+
+    Returns result in form (plot_html, csv_data)
+    """
+
+    # generate prob. distribution histogram & return embed
+    labels = ["Full Image"]
+    for i in range(len(ptemps_list[1:])):
+        labels.append(f"Firebrand {i+1}")
+    labels.reverse()
+
+    fig = ff.create_distplot(
+        ptemps_list,
+        group_labels=labels, 
+        show_rug=False,
+        show_hist=False,
+    )
+    fig.update_layout(
+        autosize=False,
+        width=800,
+        height=600,
+    )
+    fig.update_xaxes(
+        title_text="Temperature (°C)",
+    )
+    fig.update_yaxes(
+        title_text="Probability (1/°C)",
+    )
+    freq_plot = fig.to_html()
+
+    # create csv-formatted stuff
+    csvstrs = []
+    plot_data=fig.to_dict()
+    for i in range(len(plot_data["data"])):
+        x_data = plot_data["data"][i]["x"]
+        y_data = plot_data["data"][i]["y"]
+
+        tdata = [["Temperature", "Frequency"]]
+        for i in range(len(x_data)):
+            r = []
+            r.append(x_data[i])
+            r.append(y_data[i])
+            tdata.append(r)
+
+        csvstr = pd.DataFrame(tdata).to_csv(index=False, header=False)
+        csvstrs.append(csvstr)
+
+    return (
+        freq_plot,
+        csvstrs
+    )

ratio_pyrometry.py

@@ -3,6 +3,7 @@ from multiprocessing.sharedctypes import Value
 import cv2 as cv
 import numpy as np
 from numba import jit
+from skimage import measure
 
 @jit(nopython=True)
 def rg_ratio_normalize(
@@ -85,11 +86,104 @@ def ratio_pyrometry_pipeline(
     smoothing_radius: int,
     key_entries: int,
     eqn_scaling_factor: float,
+    # firebrand detection
+    firebrand_min_intensity_threshold: float,
+    firebrand_min_area: float
 ):
-
     # read image & crop
     img_orig = cv.imdecode(file_bytes, cv.IMREAD_UNCHANGED)
 
+    # ---------------------------------------------------------
+    # -- Firebrand detection
+    # ---------------------------------------------------------
+
+    img = cv.copyMakeBorder(
+        img_orig, 
+        20, 
+        20, 
+        20, 
+        20, 
+        cv.BORDER_CONSTANT, 
+        value=0
+    )
+
+    retval, thresh_gray = cv.threshold(img, firebrand_min_intensity_threshold, 255, cv.THRESH_BINARY)
+
+    kernel = np.ones((7, 7), np.uint8)
+    image = cv.morphologyEx(thresh_gray, cv.MORPH_CLOSE, kernel, iterations=1)
+    gray = cv.cvtColor(image, cv.COLOR_BGR2GRAY)
+
+    retval, gray = cv.threshold(gray, 0, 255, cv.THRESH_BINARY)
+
+    contours = measure.find_contours(array=gray, level=100)
+
+    def calculate_area(countour):
+        c = np.expand_dims(countour.astype(np.float32), 1)
+        c = cv.UMat(c)
+        
+        return cv.contourArea(c)
+
+    individual_firebrands = []
+
+    for contour in contours:
+        if calculate_area(contour) > firebrand_min_area:
+            mask = np.zeros(img.shape[0:2], dtype='uint8')
+            cv.fillPoly(mask, pts=np.int32([np.flip(contour, 1)]), color=(255,255,255))
+
+            retval, mask = cv.threshold(mask, 0, 255, cv.THRESH_BINARY)
+
+            #apply the mask to the img
+            masked = cv.bitwise_and(img, img, mask=mask)
+
+            masked_ratio_rg, ptemps_indiv = rg_ratio_normalize(
+                masked,
+                I_Darkcurrent,
+                f_stop,
+                exposure_time,
+                ISO,
+                MIN_TEMP,
+                MAX_TEMP,
+                eqn_scaling_factor,
+            )
+
+            # build & apply smoothing conv kernel
+            k = []
+            for i in range(smoothing_radius):
+                k.append([1/(smoothing_radius**2) for i in range(smoothing_radius)])
+            kernel = np.array(k)
+
+            masked_ratio_rg = cv.filter2D(src=masked_ratio_rg, ddepth=-1, kernel=kernel)
+
+            # write colormapped image
+            masked_ratio_rg_jet = cv.applyColorMap(masked_ratio_rg, cv.COLORMAP_JET)
+
+            # Generate key
+            step = (MAX_TEMP - MIN_TEMP) / (key_entries-1)
+            temps = []
+            key_img_arr = [[]]
+
+            for i in range(key_entries):
+                res_temp = MIN_TEMP + (i * step)
+                res_color = scale_temp(res_temp, MIN_TEMP, MAX_TEMP)
+                temps.append(math.floor(res_temp))
+                key_img_arr[0].append([res_color, res_color, res_color])
+
+            key_img = np.array(key_img_arr).astype(np.uint8)
+            key_img_jet = cv.applyColorMap(key_img, cv.COLORMAP_JET)
+
+            tempkey = {}
+
+            for i in range(len(temps)):
+                c = key_img_jet[0][i]
+                tempkey[temps[i]] = f"rgb({c[2]}, {c[1]}, {c[0]})"
+
+            individual_firebrands.append({
+                "img_data": masked_ratio_rg_jet,
+                "legend": tempkey,
+                "ptemps": ptemps_indiv
+            })
+
+
     img, ptemps = rg_ratio_normalize(
         img_orig,
         I_Darkcurrent,
@@ -112,7 +206,9 @@ def ratio_pyrometry_pipeline(
     # write colormapped image
     img_jet = cv.applyColorMap(img, cv.COLORMAP_JET)
 
-    # --- Generate temperature key ---
+    # ---------------------------------------------------------
+    # -- Generate temperature key
+    # ---------------------------------------------------------
 
     # Generate key
     step = (MAX_TEMP - MIN_TEMP) / (key_entries-1)
@@ -133,4 +229,4 @@ def ratio_pyrometry_pipeline(
         tempkey[temps[i]] = f"rgb({c[2]}, {c[1]}, {c[0]})"
 
     # original, transformed, legend
-    return img_orig, img_jet, tempkey, ptemps
+    return img_orig, img_jet, tempkey, ptemps, individual_firebrands

size_projection.py

@@ -9,9 +9,9 @@ import matplotlib.pyplot as plt
 from skimage import measure, morphology, color, segmentation
 import io
 
-def get_projected_area(image, area_threshold, display_threshold):
+def get_projected_area(image, area_threshold, display_threshold, paper_width, paper_height):
     total_px = image.size
-    total_mm = 60322.46
+    total_mm = paper_width * paper_height * 25.4
 
     output = []
     original = cv.imdecode(image, cv.IMREAD_UNCHANGED)

static/app.css

@@ -28,6 +28,11 @@ html {
     margin-left: 3rem;
 }
 
+.content {
+    display: flex;
+    margin: 0rem 1rem;
+}
+
 .form {
     display: flex;
     flex-direction: column;

static/img_preview.js

@@ -1,11 +0,0 @@
-function setOnchange() {
-    let imgPreview = document.getElementById('img-preview');
-    let imgUpload = document.getElementById('img-upload');
-    imgUpload.onchange = event => {
-        const [file] = imgUpload.files;
-        if (file) {
-            console.log(file)
-            imgPreview.src = URL.createObjectURL(file);
-        }
-    };
-}

static/js/csv_download.js

@@ -0,0 +1,28 @@
+function saveData(csvStr, filename) {
+    // string rep
+    // var csvStr = "";
+    // for (let r = 0; r < csvData.length; r++) {
+    //     let row = csvData[r]
+    //     for (let c = 0; c < row.length; c++) {
+    //         let item = row[c]
+    //         csvStr += item;
+    //         if (c < row.length - 1)
+    //             csvStr += ",";
+    //     }
+    //     if (r < csvStr.length - 1)
+    //         csvStr += "\r\n";
+    // }
+
+    // define data blob
+    var data = new Blob([csvStr]);
+
+    // create & click temp link
+    // slightly modded https://stackoverflow.com/a/15832662
+    var link = document.createElement("a");
+    link.download = filename;
+    link.href = URL.createObjectURL(data);
+    document.body.appendChild(link);
+    link.click();
+    document.body.removeChild(link);
+    delete link;
+}

static/js/img_preview.js

@@ -0,0 +1,9 @@
+let imgPreview = document.getElementById('img-preview');
+let imgUpload = document.getElementById('img-upload');
+imgUpload.onchange = event => {
+    const [file] = imgUpload.files;
+    if (file) {
+        console.log(file)
+        imgPreview.src = URL.createObjectURL(file);
+    }
+};

templates/base.html

@@ -2,6 +2,8 @@
 <head>
     <title>Pyrometry Application</title>
     <link rel="stylesheet" href="{{ url_for('static', filename='app.css') }}">
+    {% block head %}
+    {% endblock %}
 </head>
 <body>
     <div class="navbar">
@@ -12,7 +14,9 @@
         </div>
     </div>
     <br>
+    <div class="content">
     {% block content required %}
     {% endblock %}
+    </div>
     <br><br><br>
 </body>

templates/index.html

@@ -1,4 +1,8 @@
 {% extends "base.html" %}
+
+{% block head %}
+{% endblock %}
+
 {% block content %} 
 <form action="/ratio_pyro" method="POST" enctype="multipart/form-data">
     <h2>Ratio Pyrometry Interface</h2>
@@ -44,6 +48,16 @@
     <input type="number" name="equation_scaling_factor" value="0.55" step="0.001"/>
     <br>
 
+    <h4>Firebrand Detection Settings</h4>
+
+    <label for="intensity_threshold">intensity threshold (0-255)</label>
+    <input type="number" name="intensity_threshold" value="250", step="0.1"/>
+    <br>
+
+    <label for="min_area">min area (removes small particles)</label>
+    <input type="number" name="min_area" value="115", step="0.1"/>
+    <br>
+
     <h4>Output Settings</h4>
 
     <label for="smoothing_radius">Smoothing Radius (px)</label>
@@ -58,17 +72,7 @@
     <input type="submit" value="Generate Heatmap"/>
 </form>
 
-<!-- <script src="/img_preview.js" onload="setOnChange()"></script> -->
-<script>
-    let imgPreview = document.getElementById('img-preview');
-    let imgUpload = document.getElementById('img-upload');
-    imgUpload.onchange = event => {
-        const [file] = imgUpload.files;
-        if (file) {
-            console.log(file)
-            imgPreview.src = URL.createObjectURL(file);
-        }
-    };
-</script>
+<!-- Image Preview -->
+<script src="/s/js/img_preview.js"></script>
 
 {% endblock %} 

templates/projected-area-results.html

@@ -11,13 +11,13 @@
     <table style="background: #f1f1f1">
         <tr>
             <th class="legend-heading">Index</th>
-            <th class="legend-heading">Area</th>
+            <th class="legend-heading">Area (mm²)</th>
             <th class="legend-heading">Copy</th>
         </tr>
         {% for item in dtable %}
         <tr>
             <td class="legend-cell">{{ item.0 }}</td>
-            <td class="legend-cell">{{ item.1 }} mm²</td>
+            <td class="legend-cell">{{ item.1 }}</td>
             <td>
                 <button onclick="() => navigator.clipboard.writeText('{{ item.1 }}')">Copy</button>
             </td>

templates/projected-area.html

@@ -14,26 +14,25 @@
     <label for="area_threshold">Area threshold (to remove dust particles) in px</label>
     <input type="number" name="area_threshold" value="250"/>
     <br>
+    <br>
 
     <label for="min_display_threshold">Minimum display threshold (in px)</label>
     <input type="number" name="min_display_threshold" value="300"/>
     <br>
+    <br>
+
+    <label for="min_display_threshold">Page size</label>
+    <input type="number" name="paper_width" value="8.5"/>
+    X
+    <input type="number" name="paper_height" value="11"/> inches
+    <br>
+    <br>
 
     <br>
     <input type="submit" value="Generate Projected Sizes"/>
 </form>
 
-<!-- <script src="/img_preview.js" onload="setOnChange()"></script> -->
-<script>
-    let imgPreview = document.getElementById('img-preview');
-    let imgUpload = document.getElementById('img-upload');
-    imgUpload.onchange = event => {
-        const [file] = imgUpload.files;
-        if (file) {
-            console.log(file)
-            imgPreview.src = URL.createObjectURL(file);
-        }
-    };
-</script>
+<!-- Image Preview -->
+<script src="/s/js/img_preview.js"></script>
 
 {% endblock %}

templates/pyrometry-results.html

@@ -1,6 +1,15 @@
 {% extends "base.html" %}
+
+{% block head %}
+<script src="/s/js/csv_download.js"></script>
+{% endblock %}
+
 {% block content %} 
 
+<div style="display:flex; flex-direction: column;">
+
+<h2>General Results</h2>
+
 <table class="img-table">
     <tr>
         <th class="img-table-heading">Input Image</th>
@@ -33,8 +42,58 @@
         </td>
     </tr>
 </table>
+
+
+<h2>Individual Firebrands</h2>
+
+<table>
+    <tr>
+        <th>Output Heatmap</th>
+        <th>Legend</th>
+    </tr>
+
+    {% for item in individual_firebrands %}
+    <tr>
+        {# output heatmap #}
+        <td>
+            <img class="img-out" src="data:image/png;base64,{{ item['img_data'] }}" alt="result image">
+        </td>
+
+        </td>
+        {# legend #}
+        <td>
+            <h3>Firebrand {{ loop.index }}</h3>
+            <table class="legend" id="legend">
+                <tr>
+                    <th class="legend-heading">Color</th>
+                    <th class="legend-heading">Temperature</th>
+                </tr>
+                {% for temp, color in item["legend"].items() %}
+                <tr>
+                    <td class="legend-cell"><div style="width:30px;height:20px;background-color:{{ color }};"></div></td>
+                    <td class="legend-cell">{{ temp }}°C</td>
+                </tr>
+                {% endfor %}
+            </table>
+        </td>
+    </tr>
+    {% endfor %}
+
+</table>
+
+<br>
 {# Temperature Frequency Plot #}
+<div style="display: flex; flex-direction: row; align-items: center;">
 <strong>Temperature Distribution</strong>
+<button 
+style="width: 10rem; height: 2rem; margin-left: 1rem;" 
+onclick="saveData(`{{csv_data}}`, 'temperature-data.csv')">Download Data as CSV</button>
+</div>
 {{ freq_plot | safe }}
 
+
+<!-- Firebrands: {{ individual_firebrands }} -->
+
+</div>
+
 {% endblock %}