Licence Plate Recognition with OpenHAB and OpenALPR

Tutorials & Examples
,
1

Purpose: Perform licence plate recognition from a camera and store the plate numbers in a database for further processing.

Setup:

  • Dahua IPC-HFW5231E-Z12 camera (2MP 12x zoom). Dahua’s IVS / Motion trigger is set up to trigger the event with ipcamera binding
  • BlueIris continuously records the video stream from the camera and it serves snapshot images (among other things) on its own http server
  • OpenALPR-HTTP-Wrapper docker
  • ipcamera binding connects to the dahua camera directly for event alerts to openhab
  • Jruby for scripting, image_voodoo gem for cropping, jdbc-sqlite3 + sequel gems for sqlite access
  • The captured images are saved in OPENHAB_CONF/scripts/plates/ and kept for 30 days.
  • The resulting data (timestamp, recognised plate number) is saved in an sqlite database and a plain text

Blueiris’ role for this sc is simply as a source for getting the snapshots from the camera. I tried getting the snapshot directly from the camera, but it seems to be too slow and as a result, the vehicle had already disappeared from the camera.

Items:

Group                    Street_Camera                     "Street Camera"       <camera>  (gOutdoor)                              ["Camera"]  {snapshot_url="http://192.168.1.15/image/lpr?h=1080"}
String                   Street_Camera_EventData                                           (Street_Camera, CameraEventData)        ["Status"]  {channel="ipcamera:dahua:street:lastEventData"}

Note the metadata snapshot_url could just be set on the Street_Camera_EventData to make things easier. Just adjust the code accordingly.

It’s stored as item metadata to support multiple cameras, each will have its own unique snapshot url.

Code:

# frozen_string_literal: true

require 'openhab'
require 'json'
require 'net/http'

gemfile do
  source 'https://rubygems.org'
  gem 'image_voodoo'
  gem 'jdbc-sqlite3'
  gem 'sequel'
end

PLATE_PATH = OpenHAB.conf_root / 'scripts' / 'plates'
PLATE_FILE = PLATE_PATH.parent / 'plates.txt'
PLATE_DB = PLATE_PATH.parent / 'plates.db'
OPENALPR_URI = URI('http://192.168.1.10:5000/detect')

# Add more cameras here
CAMERA_IDS = { Street_Camera => 0 }.freeze

DB = Sequel.jdbc("sqlite:#{PLATE_DB}")

DB.create_table? :captures do
  primary_key :capture_id
  DateTime :timestamp, default: Sequel::CURRENT_TIMESTAMP, index: true
  Integer :camera_id, default: 0, index: true
  String :plate, size: 10, index: true
  String :snapshot_file
end

DB.create_table? :plates do
  String :plate, size: 10, primary_key: true
  String :notes
end

# This table is to manually correct some common mistakes of familiar plates, e.g. 338ZZZ -> 388ZZZ
DB.create_table? :plate_corrections do
  String :raw_plate, size: 10, primary_key: true
  String :plate, size: 10
end

PLATES = DB[:plates]
CAPTURES = DB[:captures]
PLATE_CORRECTIONS = DB[:plate_corrections]

#
# Holds a snapshot image with timestamp
#
class Snapshot
  attr_reader :image, :timestamp, :source

  @@camera_mutex = Mutex.new

  # Initialize with an image object
  def initialize(image, source)
    @timestamp = Time.now
    @image = image
    @source = source
  end

  # take a snapshot from the given camera
  def self.capture(camera)
    url = camera.meta['snapshot_url'].value
    image = @@camera_mutex.synchronize { Net::HTTP.get(URI(url)) }
    Snapshot.new(image, camera)
  end

  # if necessary, calls lpr to recognize the plate
  def plate
    @plate ||= lpr(@image) || ''
  end

  # Return true if we have a valid plate
  def plate?
    !plate.empty?
  end

  def to_s
    "#{timestamp} #{@plate}"
  end
end

#
# Submit the given image to openalpr-http-wrapper service and return the JSON data
#
# @param [Binary] image the binary jpg image data to process
#
# @return [Hash] the returned json data as a Ruby hash, or nil if unsuccessful
#
def openalpr(image)
  uri = OPENALPR_URI
  req = Net::HTTP::Post.new(uri)
  req.set_form([['upload', image, { filename: 'image.jpg' }]], 'multipart/form-data')
  res = Net::HTTP.start(uri.hostname, uri.port) { |http| http.request(req) }
  JSON.parse(res.body) if res.is_a? Net::HTTPSuccess
end

#
# Crop the given image
#
# @param [Image] image the binary image data
# @param [Array] rect rectangle to crop, an array with 4 elements: left, top, right, bottom
#
# @return [Image] The cropped image
#
def crop_image(image, rect)
  ImageVoodoo.with_bytes(image).with_crop(*rect).bytes('jpg')
end

#
# Perform the LPR with the given image and
# return the plate with the highest confidence
#
# This will also crop the bottom of the image to remove
# The timestamp that's baked in by the camera
#
# @param [Binary] image the binary jpg image data to process
#
# @return [String] The recognised plate or nil
#
def lpr(image)
  # crop the timestamp on the lower part of the image
  image = crop_image(image, [0, 0, 1920, 1000])

  result = openalpr(image)
  result&.dig('results')&.sort_by { |res| res['confidence'] }&.last&.dig('plate')
rescue StandardError => e
  logger.error e.message
  nil
end

# { camera_id => array of Snapshot }
@snapshots = Hash.new { |hash, key| hash[key] = [] }
@snapshot_mutex = Mutex.new

@processing_queue = []
@processing_mutex = Mutex.new

#
# Save the given snapshot
#   Append the plate to a text file
#
# @param [Snapshot] snapshot The snapshot object to save
#
def save_snapshot(snapshot)
  time_str = snapshot.timestamp.strftime('%Y-%m-%d %k:%M:%S.%L')
  lplate = snapshot.plate
  filename = "#{time_str}-#{lplate}.jpg"
  logger.info { "Saving recognised plate: #{time_str}: #{snapshot.image.length} #{lplate}" }

  File.write(PLATE_FILE, "#{time_str} #{lplate}\n", mode: 'a') unless lplate.empty?
  File.write(PLATE_PATH / filename, snapshot.image)
  CAPTURES.insert timestamp: snapshot.timestamp,
                  plate: snapshot.plate,
                  snapshot_file: filename,
                  camera_id: CAMERA_IDS[snapshot.source]
end

#
# Given an array of snapshots objects
# Apply LPR and return the longest plate number
#
# @param [Array] snapshots An array of snapshot objects
#
# @return [Snapshot] One snapshot class containing the recognised plate number
#
def best_snapshot(snapshots)
  snapshots
    .select(&:plate?) # this will trigger the lpr on each snapshot
    .group_by(&:plate)
    .max_by { |plate, _snaps| plate.length }
    &.pop&.first
end

#
# Process everything in the processing queue and empty it
#
def process_queue
  snapshots = nil
  @processing_mutex.synchronize do
    snapshots = @processing_queue
    @processing_queue = []
  end
  snapshot = best_snapshot(snapshots)
  save_snapshot snapshot if snapshot
end

#
# Add a set of snapshots to the processing queue
# And start a re-settable timer to process the queue shortly
# This will process multiple triggers of the snapshots
# to remove duplicates of the same car triggering multiple times
#
# @param [Array] snaps An array of Snapshot objects
#
def add_snapshots(snaps)
  @processing_mutex.synchronize { @processing_queue.concat snaps }
  after(3.seconds, :id => :process_queue) { process_queue }
end

#
# Get multiple snapshots 50ms apart and save them into
# the processing queue
#
# @param [GenericItem] camera The Camera item (Semantic Equipment)
# @param [Integer] count how many snapshots to take
# @param [Duration] interval the interval of the snapshots
# @param [Block] The block to be invoked once all the snapshots have been taken, passing the snapshots array
#
#
def initiate_lpr(camera, num_snapshots: 3, interval: 50.ms)
  after(0.ms) do |timer|
    next if timer.cancelled?

    snaps = nil
    @snapshot_mutex.synchronize do
      last_snapshot = @snapshots[camera].length >= num_snapshots - 1

      timer.reschedule(interval) unless last_snapshot

      time = Time.now
      logger.info { "Getting snapshot: #{@snapshots[camera].length + 1} #{time.strftime('%Y-%m-%d %k:%M:%S.%L')}" }
      @snapshots[camera] << Snapshot.capture(camera)

      snaps = @snapshots.delete(camera) if last_snapshot
    end
    add_snapshots(snaps) if snaps
  end
end

#
# Check whether a vehicle is detected
# The logic here highly depends on the type of camera in use and
# what data it provides. This particular implementation is for Dahua cameras
#
# @param [Hash] data The JSON data from ipcameras binding
#
# @return [Boolean] true if the data indicates that a vehicle may be detected
#
def vehicle_detected?(data)
  return false unless data

  data['Action'] == 'Start' || data['RegionName'] || data.dig('Object', 'ObjectType') == 'Vehicle'
end

rule 'LPR: Capture' do
  # updated Street_Camera_LastMotionType # , to: 'lineCrossingAlarm'
  updated Street_Camera_EventData
  # delay 100.ms
  run do |event|
    if logger.debug_enabled? && event.respond_to?(:state)
      logger.debug("Street Camera: #{Street_Camera_LastMotionType} #{Street_Camera_EventData}")
    end
    json = event.item.state.to_s
    data = JSON.parse(json)
    camera = event.item.equipment
    initiate_lpr(camera) if vehicle_detected?(data)
  end
end

#
# Return the file age in days
#
# @param [String] filename
#
# @return [Float] Number of days since the file was created
#
def file_age(filename)
  (Time.now - File.ctime(filename)) / 86_400
end

rule 'LPR: Clean up old snapshots' do
  every :day, at: '3am'
  # on_start
  run do
    count = Dir.glob(PLATE_PATH / '*.jpg').select { |f| file_age(f) > 30 }.map { |f| File.delete(f) }.count
    logger.info("Deleted #{count} old LPR snapshot images ") if count.positive?
  end
end

# uncomment to manually trigger it for testing
# initiate_lpr(Street_Camera)
6 Likes
2

This topic was automatically closed 41 days after the last reply. New replies are no longer allowed.