Is there a way to extract depth from the google street view api?

Question

I want to extract depth images using the google street view api. I am using python, and I am able to query rgb images, but I can't seem to obtain depth images.

I came across this: https://github.com/proog128/GSVPanoDepth.js

So, it seems depth should be available, but I don't see options for depth looking at the google street view api documentation. Is depth available via the google api?

Answer 1

Depth information is available as a grayscale image that represents the distance from the camera to each pixel in the image. To retrieve the depth map, you can use the Google Street View Image API and include the depth parameter in the request.

The depth_map parameter is set to true, which will return a depth map along with the Street View image. You can then use this depth map data to extract depth information.

https://maps.googleapis.com/maps/api/streetview?size=600x400&location=40.720032,-73.988354&heading=235&pitch=10&key=YOUR_API_KEY&depth_map=true

To do this with python you can try this

import requests
import io
from PIL import Image

# Set API endpoint URL
endpoint = 'https://maps.googleapis.com/maps/api/streetview'

# Set API parameters
params = {
    'size': '600x400',
    'location': '40.720032,-73.988354',
    'heading': '235',
    'pitch': '10',
    'key': 'YOUR_API_KEY',
    'depth_map': 'true'
}

# Send GET request to API endpoint and retrieve response
response = requests.get(endpoint, params=params)

# Extract image and depth map from response
image_bytes = response.content
depth_map_bytes = response.headers.get('X-Goog-Depthmap-Data')

# Convert image bytes to PIL Image object
image = Image.open(io.BytesIO(image_bytes))

# Show image
image.show()

# Print depth map data
print(depth_map_bytes)

Answer 2

Expanding on this Convert depth map (base64) of google street view to image This solution seems to work. However, I can't figure out how to adjust the parameters other than the coordinates.

import base64
import zlib
import numpy as np
import struct
import matplotlib.pyplot as plt
import requests
import json

def parse(b64_string):
    # fix the 'inccorrect padding' error. The length of the string needs to be divisible by 4.
    b64_string += "=" * ((4 - len(b64_string) % 4) % 4)
    # convert the URL safe format to regular format.
    data = b64_string.replace("-", "+").replace("_", "/")

    data = base64.b64decode(data)  # decode the string
    # data = zlib.decompress(data)  # decompress the data
    return np.array([d for d in data])


def parseHeader(depthMap):
    return {
        "headerSize": depthMap[0],
        "numberOfPlanes": getUInt16(depthMap, 1),
        "width": getUInt16(depthMap, 3),
        "height": getUInt16(depthMap, 5),
        "offset": getUInt16(depthMap, 7),
    }


def get_bin(a):
    ba = bin(a)[2:]
    return "0" * (8 - len(ba)) + ba


def getUInt16(arr, ind):
    a = arr[ind]
    b = arr[ind + 1]
    return int(get_bin(b) + get_bin(a), 2)


def getFloat32(arr, ind):
    return bin_to_float("".join(get_bin(i) for i in arr[ind : ind + 4][::-1]))


def bin_to_float(binary):
    return struct.unpack("!f", struct.pack("!I", int(binary, 2)))[0]


def parsePlanes(header, depthMap):
    indices = []
    planes = []
    n = [0, 0, 0]

    for i in range(header["width"] * header["height"]):
        indices.append(depthMap[header["offset"] + i])

    for i in range(header["numberOfPlanes"]):
        byteOffset = header["offset"] + header["width"] * header["height"] + i * 4 * 4
        n = [0, 0, 0]
        n[0] = getFloat32(depthMap, byteOffset)
        n[1] = getFloat32(depthMap, byteOffset + 4)
        n[2] = getFloat32(depthMap, byteOffset + 8)
        d = getFloat32(depthMap, byteOffset + 12)
        planes.append({"n": n, "d": d})

    return {"planes": planes, "indices": indices}


def computeDepthMap(header, indices, planes):

    v = [0, 0, 0]
    w = header["width"]
    h = header["height"]

    depthMap = np.empty(w * h)

    sin_theta = np.empty(h)
    cos_theta = np.empty(h)
    sin_phi = np.empty(w)
    cos_phi = np.empty(w)

    for y in range(h):
        theta = (h - y - 0.5) / h * np.pi
        sin_theta[y] = np.sin(theta)
        cos_theta[y] = np.cos(theta)

    for x in range(w):
        phi = (w - x - 0.5) / w * 2 * np.pi + np.pi / 2
        sin_phi[x] = np.sin(phi)
        cos_phi[x] = np.cos(phi)

    for y in range(h):
        for x in range(w):
            planeIdx = indices[y * w + x]

            v[0] = sin_theta[y] * cos_phi[x]
            v[1] = sin_theta[y] * sin_phi[x]
            v[2] = cos_theta[y]

            if planeIdx > 0:
                plane = planes[planeIdx]
                t = np.abs(
                    plane["d"]
                    / (
                        v[0] * plane["n"][0]
                        + v[1] * plane["n"][1]
                        + v[2] * plane["n"][2]
                    )
                )
                depthMap[y * w + (w - x - 1)] = t
            else:
                depthMap[y * w + (w - x - 1)] = 9999999999999999999.0
    return {"width": w, "height": h, "depthMap": depthMap}






endpoint = 'https://maps.googleapis.com/maps/api/streetview/metadata'

params = {
    'location': '39.7679158,-86.1580276',
    'key': 'YOUR_API_KEY',
}

r = requests.get(endpoint, params=params)
pano_id = r.json()['pano_id']

# Set API endpoint URL
endpoint = 'https://www.google.com/maps/photometa/v1'

# Set API parameters
params = {
    'authuser': '0',
    'hl': 'en',
    'gl': 'us',
    'pb': '!1m4!1smaps_sv.tactile!11m2!2m1!1b1!2m2!1sen!2suk!3m3!1m2!1e2!2s' + pano_id + '!4m57!1e1!1e2!1e3!1e4!1e5!1e6!1e8!1e12!2m1!1e1!4m1!1i48!5m1!1e1!5m1!1e2!6m1!1e1!6m1!1e2!9m36!1m3!1e2!2b1!3e2!1m3!1e2!2b0!3e3!1m3!1e3!2b1!3e2!1m3!1e3!2b0!3e3!1m3!1e8!2b0!3e3!1m3!1e1!2b0!3e3!1m3!1e4!2b0!3e3!1m3!1e10!2b1!3e2!1m3!1e10!2b0!3e3'
}

# Send GET request to API endpoint and retrieve response
response = requests.get(endpoint, params=params, proxies=None)
# response = requests.get(endpoint)

# Extract image and depth map from response
print(response.links)
response = response.content
response = json.loads(response[4:])
s = response[1][0][5][0][5][1][2]
# decode string + decompress zip
depthMapData = parse(s)
# parse first bytes to describe data
header = parseHeader(depthMapData)
# parse bytes into planes of float values
data = parsePlanes(header, depthMapData)
# compute position and values of pixels
depthMap = computeDepthMap(header, data["indices"], data["planes"])
# process float 1D array into int 2D array with 255 values
im = depthMap["depthMap"]
im[np.where(im == max(im))[0]] = 255
if min(im) < 0:
    im[np.where(im < 0)[0]] = 0
im = im.reshape((depthMap["height"], depthMap["width"])).astype(int)
# display image
plt.imshow(im)
plt.show()